diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/docs/checktransupdate.py | 22 | ||||
| -rwxr-xr-x | tools/docs/kdoc_diff | 508 | ||||
| -rwxr-xr-x | tools/docs/kernel-doc | 49 | ||||
| -rwxr-xr-x | tools/docs/sphinx-build-wrapper | 17 | ||||
| -rwxr-xr-x | tools/lib/python/jobserver.py | 4 | ||||
| -rw-r--r-- | tools/lib/python/kdoc/c_lex.py | 662 | ||||
| -rw-r--r-- | tools/lib/python/kdoc/kdoc_files.py | 151 | ||||
| -rw-r--r-- | tools/lib/python/kdoc/kdoc_item.py | 45 | ||||
| -rw-r--r-- | tools/lib/python/kdoc/kdoc_output.py | 324 | ||||
| -rw-r--r-- | tools/lib/python/kdoc/kdoc_parser.py | 292 | ||||
| -rw-r--r-- | tools/lib/python/kdoc/kdoc_re.py | 205 | ||||
| -rw-r--r-- | tools/lib/python/kdoc/kdoc_yaml_file.py | 178 | ||||
| -rw-r--r-- | tools/lib/python/kdoc/xforms_lists.py | 153 | ||||
| -rwxr-xr-x | tools/lib/python/unittest_helper.py | 363 | ||||
| -rw-r--r-- | tools/unittests/kdoc-test-schema.yaml | 156 | ||||
| -rw-r--r-- | tools/unittests/kdoc-test.yaml | 1698 | ||||
| -rwxr-xr-x | tools/unittests/run.py | 17 | ||||
| -rwxr-xr-x | tools/unittests/test_cmatch.py | 821 | ||||
| -rwxr-xr-x | tools/unittests/test_kdoc_parser.py | 560 | ||||
| -rwxr-xr-x | tools/unittests/test_kdoc_test_schema.py | 94 | ||||
| -rwxr-xr-x | tools/unittests/test_tokenizer.py | 469 |
21 files changed, 6339 insertions, 449 deletions
diff --git a/tools/docs/checktransupdate.py b/tools/docs/checktransupdate.py index e894652369a5..d7b98753e35f 100755 --- a/tools/docs/checktransupdate.py +++ b/tools/docs/checktransupdate.py @@ -13,6 +13,8 @@ The usage is as follows: This will print all the files that need to be updated or translated in the zh_CN locale. - tools/docs/checktransupdate.py Documentation/translations/zh_CN/dev-tools/testing-overview.rst This will only print the status of the specified file. +- tools/docs/checktransupdate.py Documentation/translations/zh_CN/dev-tools +This will print the status of all files under the directory. The output is something like: Documentation/dev-tools/kfence.rst @@ -76,11 +78,11 @@ def get_origin_from_trans_smartly(origin_path, t_from_head): (2) Update the translation through commit HASH (TITLE) """ # catch flag for 12-bit commit hash - HASH = r'([0-9a-f]{12})' + hash_re = r'([0-9a-f]{12})' # pattern 1: contains "update to commit HASH" - pat_update_to = re.compile(rf'update to commit {HASH}') + pat_update_to = re.compile(rf'update to commit {hash_re}') # pattern 2: contains "Update the translation through commit HASH" - pat_update_translation = re.compile(rf'Update the translation through commit {HASH}') + pat_update_translation = re.compile(rf'Update the translation through commit {hash_re}') origin_commit_hash = None for line in t_from_head["message"]: @@ -131,7 +133,7 @@ def check_per_file(file_path): opath = get_origin_path(file_path) if not os.path.isfile(opath): - logging.error("Cannot find the origin path for {file_path}") + logging.error("Cannot find the origin path for %s", file_path) return o_from_head = get_latest_commit_from(opath, "HEAD") @@ -262,7 +264,7 @@ def main(): help='Set the logging file (default: checktransupdate.log)') parser.add_argument( - "files", nargs="*", help="Files to check, if not specified, check all files" + "files", nargs="*", help="Files or directories to check, if not specified, check all files" ) args = parser.parse_args() @@ -293,6 +295,16 @@ def main(): if args.print_missing_translations: logging.info(os.path.relpath(os.path.abspath(file), linux_path)) logging.info("No translation in the locale of %s\n", args.locale) + else: + # check if the files are directories or files + new_files = [] + for file in files: + if os.path.isfile(file): + new_files.append(file) + elif os.path.isdir(file): + # for directories, list all files in the directory and its subfolders + new_files.extend(list_files_with_excluding_folders(file, [], "rst")) + files = new_files files = list(map(lambda x: os.path.relpath(os.path.abspath(x), linux_path), files)) diff --git a/tools/docs/kdoc_diff b/tools/docs/kdoc_diff new file mode 100755 index 000000000000..1aa16bdccaa3 --- /dev/null +++ b/tools/docs/kdoc_diff @@ -0,0 +1,508 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2026: Mauro Carvalho Chehab <mchehab@kernel.org>. +# +# pylint: disable=R0903,R0912,R0913,R0914,R0915,R0917 + +""" +docdiff - Check differences between kernel‑doc output between two different +commits. + +Examples +-------- + +Compare the kernel‑doc output between the last two 5.15 releases:: + + $ kdoc_diff v6.18..v6.19 + +Both outputs are cached + +Force a complete documentation scan and clean any previous cache from +6.19 to the current HEAD:: + + $ kdoc_diff 6.19.. --full --clean + +Check differences only on a single driver since origin/main:: + + $ kdoc_diff origin/main drivers/media + +Generate an YAML file and use it to check for regressions:: + + $ kdoc_diff HEAD~ drivers/media --regression + + +""" + +import os +import sys +import argparse +import subprocess +import shutil +import re +import signal + +from glob import iglob + + +SRC_DIR = os.path.dirname(os.path.realpath(__file__)) +WORK_DIR = os.path.abspath(os.path.join(SRC_DIR, "../..")) + +KDOC_BINARY = os.path.join(SRC_DIR, "kernel-doc") +KDOC_PARSER_TEST = os.path.join(WORK_DIR, "tools/unittests/test_kdoc_parser.py") + +CACHE_DIR = ".doc_diff_cache" +YAML_NAME = "out.yaml" + +DIR_NAME = { + "full": os.path.join(CACHE_DIR, "full"), + "partial": os.path.join(CACHE_DIR, "partial"), + "no-cache": os.path.join(CACHE_DIR, "no_cache"), + "tmp": os.path.join(CACHE_DIR, "__tmp__"), +} + +class GitHelper: + """Handles all Git operations""" + + def __init__(self, work_dir=None): + self.work_dir = work_dir + + def is_inside_repository(self): + """Check if we're inside a Git repository""" + try: + output = subprocess.check_output(["git", "rev-parse", + "--is-inside-work-tree"], + cwd=self.work_dir, + stderr=subprocess.STDOUT, + universal_newlines=True) + + return output.strip() == "true" + except subprocess.CalledProcessError: + return False + + def is_valid_commit(self, commit_hash): + """ + Validate that a ref (branch, tag, commit hash, etc.) can be + resolved to a commit. + """ + try: + subprocess.check_output(["git", "rev-parse", commit_hash], + cwd=self.work_dir, + stderr=subprocess.STDOUT) + return True + except subprocess.CalledProcessError: + return False + + def get_short_hash(self, commit_hash): + """Get short commit hash""" + try: + return subprocess.check_output(["git", "rev-parse", "--short", + commit_hash], + cwd=self.work_dir, + stderr=subprocess.STDOUT, + universal_newlines=True).strip() + except subprocess.CalledProcessError: + return "" + + def has_uncommitted_changes(self): + """Check for uncommitted changes""" + try: + subprocess.check_output(["git", "diff-index", + "--quiet", "HEAD", "--"], + cwd=self.work_dir, + stderr=subprocess.STDOUT) + return False + except subprocess.CalledProcessError: + return True + + def get_current_branch(self): + """Get current branch name""" + return subprocess.check_output(["git", "branch", "--show-current"], + cwd=self.work_dir, + universal_newlines=True).strip() + + def checkout_commit(self, commit_hash, quiet=True): + """Checkout a commit safely""" + args = ["git", "checkout", "-f"] + if quiet: + args.append("-q") + args.append(commit_hash) + try: + subprocess.check_output(args, cwd=self.work_dir, + stderr=subprocess.STDOUT) + + # Double-check if branch actually switched + branch = self.get_short_hash("HEAD") + if commit_hash != branch: + raise RuntimeError(f"Branch changed to '{branch}' instead of '{commit_hash}'") + + return True + except subprocess.CalledProcessError as e: + print(f"ERROR: Failed to checkout {commit_hash}: {e}", + file=sys.stderr) + return False + + +class CacheManager: + """Manages persistent cache directories""" + + def __init__(self, work_dir): + self.work_dir = work_dir + + def initialize(self): + """Create cache directories if they don't exist""" + for dir_path in DIR_NAME.values(): + abs_path = os.path.join(self.work_dir, dir_path) + if not os.path.exists(abs_path): + os.makedirs(abs_path, exist_ok=True, mode=0o755) + + def get_commit_cache(self, commit_hash, path): + """Generate cache path for a commit""" + hash_short = GitHelper(self.work_dir).get_short_hash(commit_hash) + if not hash_short: + hash_short = commit_hash + + return os.path.join(path, hash_short) + +class KernelDocRunner: + """Runs kernel-doc documentation generator""" + + def __init__(self, work_dir, kdoc_binary): + self.work_dir = work_dir + self.kdoc_binary = kdoc_binary + self.kdoc_files = None + + def find_kdoc_references(self): + """Find all files marked with kernel-doc:: directives""" + if self.kdoc_files: + print("Using cached Kdoc refs") + return self.kdoc_files + + print("Finding kernel-doc entries in Documentation...") + + files = os.path.join(self.work_dir, 'Documentation/**/*.rst') + pattern = re.compile(r"^\.\.\s+kernel-doc::\s*(\S+)") + kdoc_files = set() + + for file_path in iglob(files, recursive=True): + try: + with open(file_path, 'r', encoding='utf-8') as fp: + for line in fp: + match = pattern.match(line.strip()) + if match: + kdoc_files.add(match.group(1)) + + except OSError: + continue + + self.kdoc_files = list(kdoc_files) + + return self.kdoc_files + + def gen_yaml(self, yaml_file, kdoc_files): + """Runs kernel-doc to generate a yaml file with man and rst.""" + cmd = [self.kdoc_binary, "--man", "--rst", "--yaml", yaml_file] + cmd += kdoc_files + + print(f"YAML regression test file will be stored at: {yaml_file}") + + try: + subprocess.check_call(cmd, cwd=self.work_dir, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + return False + + return True + + def run_unittest(self, yaml_file): + """Run unit tests with the generated yaml file""" + cmd = [KDOC_PARSER_TEST, "-q", "--yaml", yaml_file] + result = subprocess.run(cmd, cwd=self.work_dir) + + if result.returncode: + print("To check for problems, try to run it again with -v\n") + print("Use -k <regex> to filter results\n\n\t$", end="") + print(" ".join(cmd) + "\n") + + return True + + def normal_run(self, tmp_dir, output_dir, kdoc_files): + """Generate man, rst and errors, storing them at tmp_dir.""" + os.makedirs(tmp_dir, exist_ok=True) + + try: + with open(os.path.join(tmp_dir, "man.log"), "w", encoding="utf-8") as out: + subprocess.check_call([self.kdoc_binary, "--man"] + kdoc_files, + cwd=self.work_dir, + stdout=out, stderr=subprocess.DEVNULL) + + with open(os.path.join(tmp_dir, "rst.log"), "w", encoding="utf-8") as out: + with open(os.path.join(tmp_dir, "err.log"), "w", encoding="utf-8") as err: + subprocess.check_call([self.kdoc_binary, "--rst"] + kdoc_files, + cwd=self.work_dir, + stdout=out, stderr=err) + except subprocess.CalledProcessError: + return False + + if output_dir: + os.replace(tmp_dir, output_dir) + + return True + + def run(self, commit_hash, tmp_dir, output_dir, kdoc_files, is_regression, + is_end): + """Run kernel-doc on its several ways""" + if not kdoc_files: + raise RuntimeError("No kernel-doc references found") + + git_helper = GitHelper(self.work_dir) + if not git_helper.checkout_commit(commit_hash, quiet=True): + raise RuntimeError(f"ERROR: can't checkout commit {commit_hash}") + + print(f"Processing {commit_hash}...") + + if not is_regression: + return self.normal_run(tmp_dir, output_dir, kdoc_files) + + yaml_file = os.path.join(tmp_dir, YAML_NAME) + + if not is_end: + return self.gen_yaml(yaml_file, kdoc_files) + + return self.run_unittest(yaml_file) + +class DiffManager: + """Compare documentation output directories with an external diff.""" + def __init__(self, diff_tool="diff", diff_args=None): + self.diff_tool = diff_tool + # default: unified, no context, ignore whitespace changes + self.diff_args = diff_args or ["-u0", "-w"] + + def diff_directories(self, dir1, dir2): + """Compare two directories using an external diff.""" + print(f"\nDiffing {dir1} and {dir2}:") + + dir1_files = set() + dir2_files = set() + has_diff = False + + for root, _, files in os.walk(dir1): + for file in files: + dir1_files.add(os.path.relpath(os.path.join(root, file), dir1)) + for root, _, files in os.walk(dir2): + for file in files: + dir2_files.add(os.path.relpath(os.path.join(root, file), dir2)) + + common_files = sorted(dir1_files & dir2_files) + for file in common_files: + f1 = os.path.join(dir1, file) + f2 = os.path.join(dir2, file) + + cmd = [self.diff_tool] + self.diff_args + [f1, f2] + try: + result = subprocess.run( + cmd, capture_output=True, text=True, check=False + ) + if result.stdout: + has_diff = True + print(f"\n{file}") + print(result.stdout, end="") + except FileNotFoundError: + print(f"ERROR: {self.diff_tool} not found") + sys.exit(1) + + # Show files that exist only in one directory + only_in_dir1 = dir1_files - dir2_files + only_in_dir2 = dir2_files - dir1_files + if only_in_dir1 or only_in_dir2: + has_diff = True + print("\nDifferential files:") + for f in sorted(only_in_dir1): + print(f" - {f} (only in {dir1})") + for f in sorted(only_in_dir2): + print(f" + {f} (only in {dir2})") + + if not has_diff: + print("\nNo differences between those two commits") + + +class SignalHandler(): + """Signal handler class.""" + + def restore(self, force_exit=False): + """Restore original HEAD state.""" + if self.restored: + return + + print(f"Restoring original branch: {self.original_head}") + try: + subprocess.check_call( + ["git", "checkout", "-f", self.original_head], + cwd=self.git_helper.work_dir, + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + print(f"Failed to restore: {e}", file=sys.stderr) + + for sig, handler in self.old_handler.items(): + signal.signal(sig, handler) + + self.restored = True + + if force_exit: + sys.exit(1) + + def signal_handler(self, sig, _): + """Handle interrupt signals.""" + print(f"\nSignal {sig} received. Restoring original state...") + + self.restore(force_exit=True) + + def __enter__(self): + """Allow using it via with command.""" + for sig in [signal.SIGINT, signal.SIGTERM]: + self.old_handler[sig] = signal.getsignal(sig) + signal.signal(sig, self.signal_handler) + + return self + + def __exit__(self, *args): + """Restore signals at the end of with block.""" + self.restore() + + def __init__(self, git_helper, original_head): + self.git_helper = git_helper + self.original_head = original_head + self.old_handler = {} + self.restored = False + +def parse_commit_range(value): + """Handle a commit range.""" + if ".." not in value: + begin = value + end = "HEAD" + else: + begin, _, end = value.partition("..") + if not end: + end = "HEAD" + + if not begin: + raise argparse.ArgumentTypeError("Need a commit begginning") + + + print(f"Range: {begin} to {end}") + + return begin, end + + +def main(): + """Main code""" + parser = argparse.ArgumentParser(description="Compare kernel documentation between commits") + parser.add_argument("commits", type=parse_commit_range, + help="commit range like old..new") + parser.add_argument("files", nargs="*", + help="files to process – if supplied the --full flag is ignored") + + parser.add_argument("--full", "-f", action="store_true", + help="Force a full scan of Documentation/*") + + parser.add_argument("--regression", "-r", action="store_true", + help="Use YAML format to check for regressions") + + parser.add_argument("--work-dir", "-w", default=WORK_DIR, + help="work dir (default: %(default)s)") + + parser.add_argument("--clean", "-c", action="store_true", + help="Clean caches") + + args = parser.parse_args() + + if args.files and args.full: + raise argparse.ArgumentError(args.full, + "cannot combine '--full' with an explicit file list") + + work_dir = os.path.abspath(args.work_dir) + + # Initialize cache + cache = CacheManager(work_dir) + cache.initialize() + + # Validate git repository + git_helper = GitHelper(work_dir) + if not git_helper.is_inside_repository(): + raise RuntimeError("Must run inside Git repository") + + old_commit, new_commit = args.commits + + old_commit = git_helper.get_short_hash(old_commit) + new_commit = git_helper.get_short_hash(new_commit) + + # Validate commits + for commit in [old_commit, new_commit]: + if not git_helper.is_valid_commit(commit): + raise RuntimeError(f"Commit '{commit}' does not exist") + + # Check for uncommitted changes + if git_helper.has_uncommitted_changes(): + raise RuntimeError("Uncommitted changes present. Commit or stash first.") + + runner = KernelDocRunner(git_helper.work_dir, KDOC_BINARY) + + # Get files to be parsed + cache_msg = " (results will be cached)" + if args.full: + kdoc_files = ["."] + diff_type = "full" + print(f"Parsing all files at {work_dir}") + if not args.files: + diff_type = "partial" + kdoc_files = runner.find_kdoc_references() + print(f"Parsing files with kernel-doc markups at {work_dir}/Documentation") + else: + diff_type = "no-cache" + cache_msg = "" + kdoc_files = args.files + + tmp_dir = DIR_NAME["tmp"] + out_path = DIR_NAME[diff_type] + + if not args.regression: + print(f"Output will be stored at: {out_path}{cache_msg}") + + # Just in case - should never happen in practice + if not kdoc_files: + raise argparse.ArgumentError(args.files, + "No kernel-doc references found") + + original_head = git_helper.get_current_branch() + + old_cache = cache.get_commit_cache(old_commit, out_path) + new_cache = cache.get_commit_cache(new_commit, out_path) + + with SignalHandler(git_helper, original_head): + if args.clean or diff_type == "no-cache": + for cache_dir in [old_cache, new_cache]: + if cache_dir and os.path.exists(cache_dir): + shutil.rmtree(cache_dir) + + if args.regression or not os.path.exists(old_cache): + old_success = runner.run(old_commit, tmp_dir, old_cache, kdoc_files, + args.regression, False) + else: + old_success = True + + if args.regression or not os.path.exists(new_cache): + new_success = runner.run(new_commit, tmp_dir, new_cache, kdoc_files, + args.regression, True) + else: + new_success = True + + if not (old_success and new_success): + raise RuntimeError("Failed to generate documentation") + + if not args.regression: + diff_manager = DiffManager() + diff_manager.diff_directories(old_cache, new_cache) + +if __name__ == "__main__": + main() diff --git a/tools/docs/kernel-doc b/tools/docs/kernel-doc index aed09f9a54dd..d9192c3f1645 100755 --- a/tools/docs/kernel-doc +++ b/tools/docs/kernel-doc @@ -210,7 +210,6 @@ def main(): help="Enable debug messages") parser.add_argument("-M", "-modulename", "--modulename", - default="Kernel API", help="Allow setting a module name at the output.") parser.add_argument("-l", "-enable-lineno", "--enable_lineno", @@ -241,11 +240,9 @@ def main(): help=EXPORT_FILE_DESC) # - # Output format mutually-exclusive group + # Output format # - out_group = parser.add_argument_group("Output format selection (mutually exclusive)") - - out_fmt = out_group.add_mutually_exclusive_group() + out_fmt = parser.add_argument_group("Output format selection (mutually exclusive)") out_fmt.add_argument("-m", "-man", "--man", action="store_true", help="Output troff manual page format.") @@ -254,6 +251,12 @@ def main(): out_fmt.add_argument("-N", "-none", "--none", action="store_true", help="Do not output documentation, only warnings.") + out_fmt.add_argument("-y", "--yaml-file", "--yaml", + help="Stores kernel-doc output on a yaml file.") + out_fmt.add_argument("-k", "--kdoc-item", "--kdoc", action="store_true", + help="Store KdocItem inside yaml file. Ued together with --yaml.") + + # # Output selection mutually-exclusive group # @@ -324,14 +327,42 @@ def main(): from kdoc.kdoc_files import KernelFiles # pylint: disable=C0415 from kdoc.kdoc_output import RestFormat, ManFormat # pylint: disable=C0415 - if args.man: - out_style = ManFormat(modulename=args.modulename) - elif args.none: + yaml_content = set() + if args.yaml_file: out_style = None + + if args.man: + yaml_content |= {"man"} + + if args.rst: + yaml_content |= {"rst"} + + if args.kdoc_item or not yaml_content: + yaml_content |= {"KdocItem"} + else: - out_style = RestFormat() + n_outputs = 0 + + if args.man: + out_style = ManFormat(modulename=args.modulename) + n_outputs += 1 + + if args.none: + out_style = None + n_outputs += 1 + + if args.rst or n_outputs == 0: + n_outputs += 1 + out_style = RestFormat() + + if n_outputs > 1: + parser.error("Those arguments are muttually exclusive: --man, --rst, --none, except when generating a YAML file.") + + elif not n_outputs: + out_style = RestFormat() kfiles = KernelFiles(verbose=args.verbose, + yaml_file=args.yaml_file, yaml_content=yaml_content, out_style=out_style, werror=args.werror, wreturn=args.wreturn, wshort_desc=args.wshort_desc, wcontents_before_sections=args.wcontents_before_sections) diff --git a/tools/docs/sphinx-build-wrapper b/tools/docs/sphinx-build-wrapper index b7c149dff06b..1bb962202784 100755 --- a/tools/docs/sphinx-build-wrapper +++ b/tools/docs/sphinx-build-wrapper @@ -238,7 +238,12 @@ class SphinxBuilder: self.latexopts = os.environ.get("LATEXOPTS", "") if not verbose: - verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "") + try: + verbose = bool(int(os.environ.get("KBUILD_VERBOSE", 0))) + except ValueError: + # Handles an eventual case where verbosity is not a number + # like KBUILD_VERBOSE="" + verbose = False if verbose is not None: self.verbose = verbose @@ -576,7 +581,6 @@ class SphinxBuilder: """ re_kernel_doc = re.compile(r"^\.\.\s+kernel-doc::\s*(\S+)") - re_man = re.compile(r'^\.TH "[^"]*" (\d+) "([^"]*)"') if docs_dir == src_dir: # @@ -616,8 +620,7 @@ class SphinxBuilder: fp = None try: for line in result.stdout.split("\n"): - match = re_man.match(line) - if not match: + if not line.startswith(".TH"): if fp: fp.write(line + '\n') continue @@ -625,7 +628,11 @@ class SphinxBuilder: if fp: fp.close() - fname = f"{output_dir}/{match.group(2)}.{match.group(1)}" + # Use shlex here, as it handles well parameters with commas + args = shlex.split(line) + fname = f"{args[1]}.{args[2]}" + fname = fname.replace("/", " ") + fname = f"{output_dir}/{fname}" if self.verbose: print(f"Creating {fname}") diff --git a/tools/lib/python/jobserver.py b/tools/lib/python/jobserver.py index aba22c33393d..0b1ffdf9f7a3 100755 --- a/tools/lib/python/jobserver.py +++ b/tools/lib/python/jobserver.py @@ -8,14 +8,14 @@ """ Interacts with the POSIX jobserver during the Kernel build time. -A "normal" jobserver task, like the one initiated by a make subrocess would do: +A "normal" jobserver task, like the one initiated by a make subprocess would do: - open read/write file descriptors to communicate with the job server; - ask for one slot by calling:: claim = os.read(reader, 1) - - when the job finshes, call:: + - when the job finishes, call:: os.write(writer, b"+") # os.write(writer, claim) diff --git a/tools/lib/python/kdoc/c_lex.py b/tools/lib/python/kdoc/c_lex.py new file mode 100644 index 000000000000..cb95f5172448 --- /dev/null +++ b/tools/lib/python/kdoc/c_lex.py @@ -0,0 +1,662 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>. + +""" +Regular expression ancillary classes. + +Those help caching regular expressions and do matching for kernel-doc. + +Please notice that the code here may rise exceptions to indicate bad +usage inside kdoc to indicate problems at the replace pattern. + +Other errors are logged via log instance. +""" + +import logging +import re + +from copy import copy + +from .kdoc_re import KernRe + +log = logging.getLogger(__name__) + +def tokenizer_set_log(logger, prefix = ""): + """ + Replace the module‑level logger with a LoggerAdapter that + prepends *prefix* to every message. + """ + global log + + class PrefixAdapter(logging.LoggerAdapter): + """ + Ancillary class to set prefix on all message logs. + """ + def process(self, msg, kwargs): + return f"{prefix}{msg}", kwargs + + # Wrap the provided logger in our adapter + log = PrefixAdapter(logger, {"prefix": prefix}) + +class CToken(): + """ + Data class to define a C token. + """ + + # Tokens that can be used by the parser. Works like an C enum. + + COMMENT = 0 #: A standard C or C99 comment, including delimiter. + STRING = 1 #: A string, including quotation marks. + CHAR = 2 #: A character, including apostophes. + NUMBER = 3 #: A number. + PUNC = 4 #: A puntuation mark: / ``,`` / ``.``. + BEGIN = 5 #: A begin character: ``{`` / ``[`` / ``(``. + END = 6 #: A end character: ``}`` / ``]`` / ``)``. + CPP = 7 #: A preprocessor macro. + HASH = 8 #: The hash character - useful to handle other macros. + OP = 9 #: A C operator (add, subtract, ...). + STRUCT = 10 #: A ``struct`` keyword. + UNION = 11 #: An ``union`` keyword. + ENUM = 12 #: A ``struct`` keyword. + TYPEDEF = 13 #: A ``typedef`` keyword. + NAME = 14 #: A name. Can be an ID or a type. + SPACE = 15 #: Any space characters, including new lines + ENDSTMT = 16 #: End of an statement (``;``). + + BACKREF = 17 #: Not a valid C sequence, but used at sub regex patterns. + + MISMATCH = 255 #: an error indicator: should never happen in practice. + + # Dict to convert from an enum interger into a string. + _name_by_val = {v: k for k, v in dict(vars()).items() if isinstance(v, int)} + + # Dict to convert from string to an enum-like integer value. + _name_to_val = {k: v for v, k in _name_by_val.items()} + + @staticmethod + def to_name(val): + """Convert from an integer value from CToken enum into a string""" + + return CToken._name_by_val.get(val, f"UNKNOWN({val})") + + @staticmethod + def from_name(name): + """Convert a string into a CToken enum value""" + if name in CToken._name_to_val: + return CToken._name_to_val[name] + + return CToken.MISMATCH + + + def __init__(self, kind, value=None, pos=0, + brace_level=0, paren_level=0, bracket_level=0): + self.kind = kind + self.value = value + self.pos = pos + self.level = (bracket_level, paren_level, brace_level) + + def __repr__(self): + name = self.to_name(self.kind) + if isinstance(self.value, str): + value = '"' + self.value + '"' + else: + value = self.value + + return f"CToken(CToken.{name}, {value}, {self.pos}, {self.level})" + +#: Regexes to parse C code, transforming it into tokens. +RE_SCANNER_LIST = [ + # + # Note that \s\S is different than .*, as it also catches \n + # + (CToken.COMMENT, r"//[^\n]*|/\*[\s\S]*?\*/"), + + (CToken.STRING, r'"(?:\\.|[^"\\])*"'), + (CToken.CHAR, r"'(?:\\.|[^'\\])'"), + + (CToken.NUMBER, r"0[xX][\da-fA-F]+[uUlL]*|0[0-7]+[uUlL]*|" + r"\d+(?:\.\d*)?(?:[eE][+-]?\d+)?[fFlL]*"), + + (CToken.ENDSTMT, r"(?:\s+;|;)"), + + (CToken.PUNC, r"[,\.]"), + + (CToken.BEGIN, r"[\[\(\{]"), + + (CToken.END, r"[\]\)\}]"), + + (CToken.CPP, r"#\s*(?:define|include|ifdef|ifndef|if|else|elif|endif|undef|pragma)\b"), + + (CToken.HASH, r"#"), + + (CToken.OP, r"\+\+|\-\-|\->|==|\!=|<=|>=|&&|\|\||<<|>>|\+=|\-=|\*=|/=|%=" + r"|&=|\|=|\^=|[=\+\-\*/%<>&\|\^~!\?\:]"), + + (CToken.STRUCT, r"\bstruct\b"), + (CToken.UNION, r"\bunion\b"), + (CToken.ENUM, r"\benum\b"), + (CToken.TYPEDEF, r"\btypedef\b"), + + (CToken.NAME, r"[A-Za-z_]\w*"), + + (CToken.SPACE, r"\s+"), + + (CToken.BACKREF, r"\\\d+"), + + (CToken.MISMATCH,r"."), +] + +def fill_re_scanner(token_list): + """Ancillary routine to convert RE_SCANNER_LIST into a finditer regex""" + re_tokens = [] + + for kind, pattern in token_list: + name = CToken.to_name(kind) + re_tokens.append(f"(?P<{name}>{pattern})") + + return KernRe("|".join(re_tokens), re.MULTILINE | re.DOTALL) + +#: Handle C continuation lines. +RE_CONT = KernRe(r"\\\n") + +RE_COMMENT_START = KernRe(r'/\*\s*') + +#: tokenizer regex. Will be filled at the first CTokenizer usage. +RE_SCANNER = fill_re_scanner(RE_SCANNER_LIST) + + +class CTokenizer(): + """ + Scan C statements and definitions and produce tokens. + + When converted to string, it drops comments and handle public/private + values, respecting depth. + """ + + # This class is inspired and follows the basic concepts of: + # https://docs.python.org/3/library/re.html#writing-a-tokenizer + + def __init__(self, source=None): + """ + Create a regular expression to handle RE_SCANNER_LIST. + + While I generally don't like using regex group naming via: + (?P<name>...) + + in this particular case, it makes sense, as we can pick the name + when matching a code via RE_SCANNER. + """ + + # + # Store logger to allow parser classes to re-use it + # + global log + self.log = log + + self.tokens = [] + + if not source: + return + + if isinstance(source, list): + self.tokens = source + return + + # + # While we could just use _tokenize directly via interator, + # As we'll need to use the tokenizer several times inside kernel-doc + # to handle macro transforms, cache the results on a list, as + # re-using it is cheaper than having to parse everytime. + # + for tok in self._tokenize(source): + self.tokens.append(tok) + + def _tokenize(self, source): + """ + Iterator that parses ``source``, splitting it into tokens, as defined + at ``self.RE_SCANNER_LIST``. + + The interactor returns a CToken class object. + """ + + # Handle continuation lines. Note that kdoc_parser already has a + # logic to do that. Still, let's keep it for completeness, as we might + # end re-using this tokenizer outsize kernel-doc some day - or we may + # eventually remove from there as a future cleanup. + source = RE_CONT.sub("", source) + + brace_level = 0 + paren_level = 0 + bracket_level = 0 + + for match in RE_SCANNER.finditer(source): + kind = CToken.from_name(match.lastgroup) + pos = match.start() + value = match.group() + + if kind == CToken.MISMATCH: + log.error(f"Unexpected token '{value}' on pos {pos}:\n\t'{source}'") + elif kind == CToken.BEGIN: + if value == '(': + paren_level += 1 + elif value == '[': + bracket_level += 1 + else: # value == '{' + brace_level += 1 + + elif kind == CToken.END: + if value == ')' and paren_level > 0: + paren_level -= 1 + elif value == ']' and bracket_level > 0: + bracket_level -= 1 + elif brace_level > 0: # value == '}' + brace_level -= 1 + + yield CToken(kind, value, pos, + brace_level, paren_level, bracket_level) + + def __str__(self): + out="" + show_stack = [True] + + for i, tok in enumerate(self.tokens): + if tok.kind == CToken.BEGIN: + show_stack.append(show_stack[-1]) + + elif tok.kind == CToken.END: + prev = show_stack[-1] + if len(show_stack) > 1: + show_stack.pop() + + if not prev and show_stack[-1]: + # + # Try to preserve indent + # + out += "\t" * (len(show_stack) - 1) + + out += str(tok.value) + continue + + elif tok.kind == CToken.COMMENT: + comment = RE_COMMENT_START.sub("", tok.value) + + if comment.startswith("private:"): + show_stack[-1] = False + show = False + elif comment.startswith("public:"): + show_stack[-1] = True + + continue + + if not show_stack[-1]: + continue + + if i < len(self.tokens) - 1: + next_tok = self.tokens[i + 1] + + # Do some cleanups before ";" + + if tok.kind == CToken.SPACE and next_tok.kind == CToken.ENDSTMT: + continue + + if tok.kind == CToken.ENDSTMT and next_tok.kind == tok.kind: + continue + + out += str(tok.value) + + return out + + +class CTokenArgs: + """ + Ancillary class to help using backrefs from sub matches. + + If the highest backref contain a "+" at the last element, + the logic will be greedy, picking all other delims. + + This is needed to parse struct_group macros with end with ``MEMBERS...``. + """ + def __init__(self, sub_str): + self.sub_groups = set() + self.max_group = -1 + self.greedy = None + + for m in KernRe(r'\\(\d+)([+]?)').finditer(sub_str): + group = int(m.group(1)) + if m.group(2) == "+": + if self.greedy and self.greedy != group: + raise ValueError("There are multiple greedy patterns!") + self.greedy = group + + self.sub_groups.add(group) + self.max_group = max(self.max_group, group) + + if self.greedy: + if self.greedy != self.max_group: + raise ValueError("Greedy pattern is not the last one!") + + sub_str = KernRe(r'(\\\d+)[+]').sub(r"\1", sub_str) + + self.sub_str = sub_str + self.sub_tokeninzer = CTokenizer(sub_str) + + def groups(self, new_tokenizer): + r""" + Create replacement arguments for backrefs like: + + ``\0``, ``\1``, ``\2``, ... ``\{number}`` + + It also accepts a ``+`` character to the highest backref, like + ``\4+``. When used, the backref will be greedy, picking all other + arguments afterwards. + + The logic is smart enough to only go up to the maximum required + argument, even if there are more. + + If there is a backref for an argument above the limit, it will + raise an exception. Please notice that, on C, square brackets + don't have any separator on it. Trying to use ``\1``..``\n`` for + brackets also raise an exception. + """ + + level = (0, 0, 0) + + if self.max_group < 0: + return level, [] + + tokens = new_tokenizer.tokens + + # + # Fill \0 with the full token contents + # + groups_list = [ [] ] + + if 0 in self.sub_groups: + inner_level = 0 + + for i in range(0, len(tokens)): + tok = tokens[i] + + if tok.kind == CToken.BEGIN: + inner_level += 1 + + # + # Discard first begin + # + if not groups_list[0]: + continue + elif tok.kind == CToken.END: + inner_level -= 1 + if inner_level < 0: + break + + if inner_level: + groups_list[0].append(tok) + + if not self.max_group: + return level, groups_list + + delim = None + + # + # Ignore everything before BEGIN. The value of begin gives the + # delimiter to be used for the matches + # + for i in range(0, len(tokens)): + tok = tokens[i] + if tok.kind == CToken.BEGIN: + if tok.value == "{": + delim = ";" + elif tok.value == "(": + delim = "," + else: + self.log.error(fr"Can't handle \1..\n on {sub_str}") + + level = tok.level + break + + pos = 1 + groups_list.append([]) + + inner_level = 0 + for i in range(i + 1, len(tokens)): + tok = tokens[i] + + if tok.kind == CToken.BEGIN: + inner_level += 1 + if tok.kind == CToken.END: + inner_level -= 1 + if inner_level < 0: + break + + if tok.kind in [CToken.PUNC, CToken.ENDSTMT] and delim == tok.value: + pos += 1 + if self.greedy and pos > self.max_group: + pos -= 1 + else: + groups_list.append([]) + + if pos > self.max_group: + break + + continue + + groups_list[pos].append(tok) + + if pos < self.max_group: + log.error(fr"{self.sub_str} groups are up to {pos} instead of {self.max_group}") + + return level, groups_list + + def tokens(self, new_tokenizer): + level, groups = self.groups(new_tokenizer) + + new = CTokenizer() + + for tok in self.sub_tokeninzer.tokens: + if tok.kind == CToken.BACKREF: + group = int(tok.value[1:]) + + for group_tok in groups[group]: + new_tok = copy(group_tok) + + new_level = [0, 0, 0] + + for i in range(0, len(level)): + new_level[i] = new_tok.level[i] + level[i] + + new_tok.level = tuple(new_level) + + new.tokens += [ new_tok ] + else: + new.tokens += [ tok ] + + return new.tokens + + +class CMatch: + """ + Finding nested delimiters is hard with regular expressions. It is + even harder on Python with its normal re module, as there are several + advanced regular expressions that are missing. + + This is the case of this pattern:: + + '\\bSTRUCT_GROUP(\\(((?:(?>[^)(]+)|(?1))*)\\))[^;]*;' + + which is used to properly match open/close parentheses of the + string search STRUCT_GROUP(), + + Add a class that counts pairs of delimiters, using it to match and + replace nested expressions. + + The original approach was suggested by: + + https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex + + Although I re-implemented it to make it more generic and match 3 types + of delimiters. The logic checks if delimiters are paired. If not, it + will ignore the search string. + """ + + + def __init__(self, regex, delim="("): + self.regex = KernRe("^" + regex + r"\b") + self.start_delim = delim + + def _search(self, tokenizer): + """ + Finds paired blocks for a regex that ends with a delimiter. + + The suggestion of using finditer to match pairs came from: + https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex + but I ended using a different implementation to align all three types + of delimiters and seek for an initial regular expression. + + The algorithm seeks for open/close paired delimiters and places them + into a stack, yielding a start/stop position of each match when the + stack is zeroed. + + The algorithm should work fine for properly paired lines, but will + silently ignore end delimiters that precede a start delimiter. + This should be OK for kernel-doc parser, as unaligned delimiters + would cause compilation errors. So, we don't need to raise exceptions + to cover such issues. + """ + + start = None + started = False + + import sys + + stack = [] + + for i, tok in enumerate(tokenizer.tokens): + if start is None: + if tok.kind == CToken.NAME and self.regex.match(tok.value): + start = i + stack.append((start, tok.level)) + started = False + + continue + + if not started: + if tok.kind == CToken.SPACE: + continue + + if tok.kind == CToken.BEGIN and tok.value == self.start_delim: + started = True + continue + + # Name only token without BEGIN/END + if i > start: + i -= 1 + yield start, i + start = None + + if tok.kind == CToken.END and tok.level == stack[-1][1]: + start, level = stack.pop() + + yield start, i + start = None + + # + # If an END zeroing levels is not there, return remaining stuff + # This is meant to solve cases where the caller logic might be + # picking an incomplete block. + # + if start and stack: + if started: + s = str(tokenizer) + log.warning(f"can't find a final end at {s}") + + yield start, len(tokenizer.tokens) + + def search(self, source): + """ + This is similar to re.search: + + It matches a regex that it is followed by a delimiter, + returning occurrences only if all delimiters are paired. + """ + + if isinstance(source, CTokenizer): + tokenizer = source + is_token = True + else: + tokenizer = CTokenizer(source) + is_token = False + + for start, end in self._search(tokenizer): + new_tokenizer = CTokenizer(tokenizer.tokens[start:end + 1]) + + if is_token: + yield new_tokenizer + else: + yield str(new_tokenizer) + + def sub(self, sub_str, source, count=0): + """ + This is similar to re.sub: + + It matches a regex that it is followed by a delimiter, + replacing occurrences only if all delimiters are paired. + + if the sub argument contains:: + + r'\0' + + it will work just like re: it places there the matched paired data + with the delimiter stripped. + + If count is different than zero, it will replace at most count + items. + """ + if isinstance(source, CTokenizer): + is_token = True + tokenizer = source + else: + is_token = False + tokenizer = CTokenizer(source) + + # Detect if sub_str contains sub arguments + + args_match = CTokenArgs(sub_str) + + new_tokenizer = CTokenizer() + pos = 0 + n = 0 + + # + # NOTE: the code below doesn't consider overlays at sub. + # We may need to add some extra unit tests to check if those + # would cause problems. When replacing by "", this should not + # be a problem, but other transformations could be problematic + # + for start, end in self._search(tokenizer): + new_tokenizer.tokens += tokenizer.tokens[pos:start] + + new = CTokenizer(tokenizer.tokens[start:end + 1]) + + new_tokenizer.tokens += args_match.tokens(new) + + pos = end + 1 + + n += 1 + if count and n >= count: + break + + new_tokenizer.tokens += tokenizer.tokens[pos:] + + if not is_token: + return str(new_tokenizer) + + return new_tokenizer + + def __repr__(self): + """ + Returns a displayable version of the class init. + """ + + return f'CMatch("{self.regex.regex.pattern}")' diff --git a/tools/lib/python/kdoc/kdoc_files.py b/tools/lib/python/kdoc/kdoc_files.py index 022487ea2cc6..ed82b6e6ab25 100644 --- a/tools/lib/python/kdoc/kdoc_files.py +++ b/tools/lib/python/kdoc/kdoc_files.py @@ -9,13 +9,14 @@ Classes for navigating through the files that kernel-doc needs to handle to generate documentation. """ -import argparse import logging import os import re from kdoc.kdoc_parser import KernelDoc +from kdoc.xforms_lists import CTransforms from kdoc.kdoc_output import OutputFormat +from kdoc.kdoc_yaml_file import KDocTestFile class GlobSourceFiles: @@ -86,11 +87,81 @@ class GlobSourceFiles: file_not_found_cb(fname) +class KdocConfig(): + """ + Stores all configuration attributes that kdoc_parser and kdoc_output + needs. + """ + def __init__(self, verbose=False, werror=False, wreturn=False, + wshort_desc=False, wcontents_before_sections=False, + logger=None): + + self.verbose = verbose + self.werror = werror + self.wreturn = wreturn + self.wshort_desc = wshort_desc + self.wcontents_before_sections = wcontents_before_sections + + if logger: + self.log = logger + else: + self.log = logging.getLogger(__file__) + + self.warning = self.log.warning + class KernelFiles(): """ Parse kernel-doc tags on multiple kernel source files. - There are two type of parsers defined here: + This is the main entry point to run kernel-doc. This class is initialized + using a series of optional arguments: + + ``verbose`` + If True, enables kernel-doc verbosity. Default: False. + + ``out_style`` + Class to be used to format output. If None (default), + only report errors. + + ``xforms`` + Transforms to be applied to C prototypes and data structs. + If not specified, defaults to xforms = CFunction() + + ``werror`` + If True, treat warnings as errors, retuning an error code on warnings. + + Default: False. + + ``wreturn`` + If True, warns about the lack of a return markup on functions. + + Default: False. + ``wshort_desc`` + If True, warns if initial short description is missing. + + Default: False. + + ``wcontents_before_sections`` + If True, warn if there are contents before sections (deprecated). + This option is kept just for backward-compatibility, but it does + nothing, neither here nor at the original Perl script. + + Default: False. + + ``logger`` + Optional logger class instance. + + If not specified, defaults to use: ``logging.getLogger("kernel-doc")`` + + ``yaml_file`` + If defined, stores the output inside a YAML file. + + ``yaml_content`` + Defines what will be inside the YAML file. + + Note: + There are two type of parsers defined here: + - self.parse_file(): parses both kernel-doc markups and ``EXPORT_SYMBOL*`` macros; - self.process_export_file(): parses only ``EXPORT_SYMBOL*`` macros. @@ -117,7 +188,12 @@ class KernelFiles(): if fname in self.files: return - doc = KernelDoc(self.config, fname) + if self.test_file: + store_src = True + else: + store_src = False + + doc = KernelDoc(self.config, fname, self.xforms, store_src=store_src) export_table, entries = doc.parse_kdoc() self.export_table[fname] = export_table @@ -153,16 +229,21 @@ class KernelFiles(): self.error(f"Cannot find file {fname}") - def __init__(self, verbose=False, out_style=None, + def __init__(self, verbose=False, out_style=None, xforms=None, werror=False, wreturn=False, wshort_desc=False, wcontents_before_sections=False, - logger=None): + yaml_file=None, yaml_content=None, logger=None): """ Initialize startup variables and parse all files. """ if not verbose: - verbose = bool(os.environ.get("KBUILD_VERBOSE", 0)) + try: + verbose = bool(int(os.environ.get("KBUILD_VERBOSE", 0))) + except ValueError: + # Handles an eventual case where verbosity is not a number + # like KBUILD_VERBOSE="" + verbose = False if out_style is None: out_style = OutputFormat() @@ -181,29 +262,36 @@ class KernelFiles(): if kdoc_werror: werror = kdoc_werror + if not logger: + logger = logging.getLogger("kernel-doc") + else: + logger = logger + # Some variables are global to the parser logic as a whole as they are # used to send control configuration to KernelDoc class. As such, # those variables are read-only inside the KernelDoc. - self.config = argparse.Namespace + self.config = KdocConfig(verbose, werror, wreturn, wshort_desc, + wcontents_before_sections, logger) - self.config.verbose = verbose - self.config.werror = werror - self.config.wreturn = wreturn - self.config.wshort_desc = wshort_desc - self.config.wcontents_before_sections = wcontents_before_sections + # Override log warning, as we want to count errors + self.config.warning = self.warning - if not logger: - self.config.log = logging.getLogger("kernel-doc") + if yaml_file: + self.test_file = KDocTestFile(self.config, yaml_file, yaml_content) else: - self.config.log = logger + self.test_file = None - self.config.warning = self.warning + if xforms: + self.xforms = xforms + else: + self.xforms = CTransforms() self.config.src_tree = os.environ.get("SRCTREE", None) # Initialize variables that are internal to KernelFiles self.out_style = out_style + self.out_style.set_config(self.config) self.errors = 0 self.results = {} @@ -246,8 +334,6 @@ class KernelFiles(): returning kernel-doc markups on each interaction. """ - self.out_style.set_config(self.config) - if not filenames: filenames = sorted(self.results.keys()) @@ -267,29 +353,28 @@ class KernelFiles(): for s in symbol: function_table.add(s) - self.out_style.set_filter(export, internal, symbol, nosymbol, - function_table, enable_lineno, - no_doc_sections) - - msg = "" if fname not in self.results: self.config.log.warning("No kernel-doc for file %s", fname) continue symbols = self.results[fname] - self.out_style.set_symbols(symbols) - for arg in symbols: - m = self.out_msg(fname, arg.name, arg) + if self.test_file: + self.test_file.set_filter(export, internal, symbol, nosymbol, + function_table, enable_lineno, + no_doc_sections) - if m is None: - ln = arg.get("ln", 0) - dtype = arg.get('type', "") + self.test_file.output_symbols(fname, symbols) - self.config.log.warning("%s:%d Can't handle %s", - fname, ln, dtype) - else: - msg += m + continue + + self.out_style.set_filter(export, internal, symbol, nosymbol, + function_table, enable_lineno, + no_doc_sections) + msg = self.out_style.output_symbols(fname, symbols) if msg: yield fname, msg + + if self.test_file: + self.test_file.write() diff --git a/tools/lib/python/kdoc/kdoc_item.py b/tools/lib/python/kdoc/kdoc_item.py index 2b8a93f79716..a7aa6e1e4c1c 100644 --- a/tools/lib/python/kdoc/kdoc_item.py +++ b/tools/lib/python/kdoc/kdoc_item.py @@ -14,7 +14,8 @@ class KdocItem: then pass into the output modules. """ - def __init__(self, name, fname, type, start_line, **other_stuff): + def __init__(self, name, fname, type, start_line, + **other_stuff): self.name = name self.fname = fname self.type = type @@ -22,15 +23,34 @@ class KdocItem: self.sections = {} self.sections_start_lines = {} self.parameterlist = [] - self.parameterdesc_start_lines = [] + self.parameterdesc_start_lines = {} self.parameterdescs = {} self.parametertypes = {} + + self.warnings = [] + # # Just save everything else into our own dict so that the output # side can grab it directly as before. As we move things into more # structured data, this will, hopefully, fade away. # - self.other_stuff = other_stuff + known_keys = { + 'declaration_start_line', + 'sections', + 'sections_start_lines', + 'parameterlist', + 'parameterdesc_start_lines', + 'parameterdescs', + 'parametertypes', + 'warnings', + } + + self.other_stuff = {} + for k, v in other_stuff.items(): + if k in known_keys: + setattr(self, k, v) # real attribute + else: + self.other_stuff[k] = v def get(self, key, default = None): """ @@ -41,6 +61,23 @@ class KdocItem: def __getitem__(self, key): return self.get(key) + def __repr__(self): + return f"KdocItem({self.name}, {self.fname}, {self.type}, {self.declaration_start_line})" + + @classmethod + def from_dict(cls, d): + """Create a KdocItem from a plain dict.""" + + cp = d.copy() + name = cp.pop('name', None) + fname = cp.pop('fname', None) + type = cp.pop('type', None) + start_line = cp.pop('start_line', 1) + other_stuff = cp.pop('other_stuff', {}) + + # Everything that’s left goes straight to __init__ + return cls(name, fname, type, start_line, **cp, **other_stuff) + # # Tracking of section and parameter information. # @@ -49,7 +86,7 @@ class KdocItem: Set sections and start lines. """ self.sections = sections - self.section_start_lines = start_lines + self.sections_start_lines = start_lines def set_params(self, names, descs, types, starts): """ diff --git a/tools/lib/python/kdoc/kdoc_output.py b/tools/lib/python/kdoc/kdoc_output.py index 4210b91dde5f..de107ab4a281 100644 --- a/tools/lib/python/kdoc/kdoc_output.py +++ b/tools/lib/python/kdoc/kdoc_output.py @@ -222,6 +222,27 @@ class OutputFormat: return None + def output_symbols(self, fname, symbols): + """ + Handles a set of KdocItem symbols. + """ + self.set_symbols(symbols) + + msg = "" + for arg in symbols: + m = self.msg(fname, arg.name, arg) + + if m is None: + ln = arg.get("ln", 0) + dtype = arg.get('type', "") + + self.config.log.warning("%s:%d Can't handle %s", + fname, ln, dtype) + else: + msg += m + + return msg + # Virtual methods to be overridden by inherited classes # At the base class, those do nothing. def set_symbols(self, symbols): @@ -368,7 +389,7 @@ class RestFormat(OutputFormat): else: self.data += f'{self.lineprefix}**{section}**\n\n' - self.print_lineno(args.section_start_lines.get(section, 0)) + self.print_lineno(args.sections_start_lines.get(section, 0)) self.output_highlight(text) self.data += "\n" self.data += "\n" @@ -492,7 +513,9 @@ class RestFormat(OutputFormat): def out_var(self, fname, name, args): oldprefix = self.lineprefix ln = args.declaration_start_line - full_proto = args.other_stuff["full_proto"] + full_proto = args.other_stuff.get("full_proto") + if not full_proto: + raise KeyError(f"Can't find full proto for {name} variable") self.lineprefix = " " @@ -580,7 +603,35 @@ class RestFormat(OutputFormat): class ManFormat(OutputFormat): - """Consts and functions used by man pages output.""" + """ + Consts and functions used by man pages output. + + This class has one mandatory parameter and some optional ones, which + are needed to define the title header contents: + + ``modulename`` + Defines the module name to be used at the troff ``.TH`` output. + + This argument is optional. If not specified, it will be filled + with the directory which contains the documented file. + + ``section`` + Usually a numeric value from 0 to 9, but man pages also accept + some strings like "p". + + Defauls to ``9`` + + ``manual`` + Defaults to ``Kernel API Manual``. + + The above controls the output of teh corresponding fields on troff + title headers, which will be filled like this:: + + .TH "{name}" {section} "{date}" "{modulename}" "{manual}" + + where ``name``` will match the API symbol name, and ``date`` will be + either the date where the Kernel was compiled or the current date + """ highlights = ( (type_constant, r"\1"), @@ -607,7 +658,21 @@ class ManFormat(OutputFormat): "%m %d %Y", ] - def __init__(self, modulename): + def modulename(self, args): + if self._modulename: + return self._modulename + + return os.path.dirname(args.fname) + + def emit_th(self, name, args): + """Emit a title header line.""" + title = name.strip() + module = self.modulename(args) + + self.data += f'.TH "{title}" {self.section} "{self.date}" ' + self.data += f'"{module}" "{self.manual}"\n' + + def __init__(self, modulename=None, section="9", manual="Kernel API Manual"): """ Creates class variables. @@ -616,7 +681,11 @@ class ManFormat(OutputFormat): """ super().__init__() - self.modulename = modulename + + self._modulename = modulename + self.section = section + self.manual = manual + self.symbols = [] dt = None @@ -632,7 +701,7 @@ class ManFormat(OutputFormat): if not dt: dt = datetime.now() - self.man_date = dt.strftime("%B %Y") + self.date = dt.strftime("%B %Y") def arg_name(self, args, name): """ @@ -647,7 +716,8 @@ class ManFormat(OutputFormat): dtype = args.type if dtype == "doc": - return self.modulename + return name +# return os.path.basename(self.modulename(args)) if dtype in ["function", "typedef"]: return name @@ -697,6 +767,185 @@ class ManFormat(OutputFormat): return self.data + def emit_table(self, colspec_row, rows): + + if not rows: + return "" + + out = "" + colspec = "\t".join(["l"] * len(rows[0])) + + out += "\n.TS\n" + out += "box;\n" + out += f"{colspec}.\n" + + if colspec_row: + out_row = [] + + for text in colspec_row: + out_row.append(f"\\fB{text}\\fP") + + out += "\t".join(out_row) + "\n_\n" + + for r in rows: + out += "\t".join(r) + "\n" + + out += ".TE\n" + + return out + + def grid_table(self, lines, start): + """ + Ancillary function to help handling a grid table inside the text. + """ + + i = start + 1 + rows = [] + colspec_row = None + + while i < len(lines): + line = lines[i] + + if KernRe(r"^\s*\|.*\|\s*$").match(line): + parts = [] + + for p in line.strip('|').split('|'): + parts.append(p.strip()) + + rows.append(parts) + + elif KernRe(r'^\+\=[\+\=]+\+\s*$').match(line): + if rows and rows[0]: + if not colspec_row: + colspec_row = [""] * len(rows[0]) + + for j in range(0, len(rows[0])): + content = [] + for row in rows: + content.append(row[j]) + + colspec_row[j] = " ".join(content) + + rows = [] + + elif KernRe(r"^\s*\+[-+]+\+.*$").match(line): + pass + + else: + break + + i += 1 + + return i, self.emit_table(colspec_row, rows) + + def simple_table(self, lines, start): + """ + Ancillary function to help handling a simple table inside the text. + """ + + i = start + rows = [] + colspec_row = None + + pos = [] + for m in KernRe(r'\=+').finditer(lines[i]): + pos.append((m.start(), m.end() - 1)) + + i += 1 + while i < len(lines): + line = lines[i] + + if KernRe(r"^\s*[\=]+[ \t\=]+$").match(line): + i += 1 + break + + elif KernRe(r'^[\s=]+$').match(line): + if rows and rows[0]: + if not colspec_row: + colspec_row = [""] * len(rows[0]) + + for j in range(0, len(rows[0])): + content = [] + for row in rows: + content.append(row[j]) + + colspec_row[j] = " ".join(content) + + rows = [] + + else: + row = [""] * len(pos) + + for j in range(0, len(pos)): + start, end = pos[j] + + row[j] = line[start:end].strip() + + rows.append(row) + + i += 1 + + return i, self.emit_table(colspec_row, rows) + + def code_block(self, lines, start): + """ + Ensure that code blocks won't be messed up at the output. + + By default, troff join lines at the same paragraph. Disable it, + on code blocks. + """ + + line = lines[start] + + if "code-block" in line: + out = "\n.nf\n" + elif line.startswith("..") and line.endswith("::"): + # + # Handle note, warning, error, ... markups + # + line = line[2:-1].strip().upper() + out = f"\n.nf\n\\fB{line}\\fP\n" + elif line.endswith("::"): + out = line[:-1] + out += "\n.nf\n" + else: + # Just in case. Should never happen in practice + out = "\n.nf\n" + + i = start + 1 + ident = None + + while i < len(lines): + line = lines[i] + + m = KernRe(r"\S").match(line) + if not m: + out += line + "\n" + i += 1 + continue + + pos = m.start() + if not ident: + if pos > 0: + ident = pos + else: + out += "\n.fi\n" + if i > start + 1: + return i - 1, out + else: + # Just in case. Should never happen in practice + return i, out + + if pos >= ident: + out += line + "\n" + i += 1 + continue + + break + + out += "\n.fi\n" + return i, out + def output_highlight(self, block): """ Outputs a C symbol that may require being highlighted with @@ -708,15 +957,46 @@ class ManFormat(OutputFormat): if isinstance(contents, list): contents = "\n".join(contents) - for line in contents.strip("\n").split("\n"): - line = KernRe(r"^\s*").sub("", line) - if not line: - continue + lines = contents.strip("\n").split("\n") + i = 0 - if line[0] == ".": - self.data += "\\&" + line + "\n" + while i < len(lines): + org_line = lines[i] + + line = KernRe(r"^\s*").sub("", org_line) + + if line: + if KernRe(r"^\+\-[-+]+\+.*$").match(line): + i, text = self.grid_table(lines, i) + self.data += text + continue + + if KernRe(r"^\=+[ \t]\=[ \t\=]+$").match(line): + i, text = self.simple_table(lines, i) + self.data += text + continue + + if line.endswith("::") or KernRe(r"\.\.\s+code-block.*::").match(line): + i, text = self.code_block(lines, i) + self.data += text + continue + + if line[0] == ".": + self.data += "\\&" + line + "\n" + i += 1 + continue + + # + # Handle lists + # + line = KernRe(r'^[-*]\s+').sub(r'.IP \[bu]\n', line) + line = KernRe(r'^(\d+|a-z)[\.\)]\s+').sub(r'.IP \1\n', line) else: - self.data += line + "\n" + line = ".PP\n" + + i += 1 + + self.data += line + "\n" def out_doc(self, fname, name, args): if not self.check_doc(name, args): @@ -724,7 +1004,7 @@ class ManFormat(OutputFormat): out_name = self.arg_name(args, name) - self.data += f'.TH "{self.modulename}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.emit_th(out_name, args) for section, text in args.sections.items(): self.data += f'.SH "{section}"' + "\n" @@ -734,7 +1014,7 @@ class ManFormat(OutputFormat): out_name = self.arg_name(args, name) - self.data += f'.TH "{name}" 9 "{out_name}" "{self.man_date}" "Kernel Hacker\'s Manual" LINUX' + "\n" + self.emit_th(out_name, args) self.data += ".SH NAME\n" self.data += f"{name} \\- {args['purpose']}\n" @@ -780,7 +1060,7 @@ class ManFormat(OutputFormat): def out_enum(self, fname, name, args): out_name = self.arg_name(args, name) - self.data += f'.TH "{self.modulename}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.emit_th(out_name, args) self.data += ".SH NAME\n" self.data += f"enum {name} \\- {args['purpose']}\n" @@ -813,7 +1093,7 @@ class ManFormat(OutputFormat): out_name = self.arg_name(args, name) full_proto = args.other_stuff["full_proto"] - self.data += f'.TH "{self.modulename}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.emit_th(out_name, args) self.data += ".SH NAME\n" self.data += f"{name} \\- {args['purpose']}\n" @@ -830,11 +1110,11 @@ class ManFormat(OutputFormat): self.output_highlight(text) def out_typedef(self, fname, name, args): - module = self.modulename + module = self.modulename(args) purpose = args.get('purpose') out_name = self.arg_name(args, name) - self.data += f'.TH "{module}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.emit_th(out_name, args) self.data += ".SH NAME\n" self.data += f"typedef {name} \\- {purpose}\n" @@ -844,12 +1124,12 @@ class ManFormat(OutputFormat): self.output_highlight(text) def out_struct(self, fname, name, args): - module = self.modulename + module = self.modulename(args) purpose = args.get('purpose') definition = args.get('definition') out_name = self.arg_name(args, name) - self.data += f'.TH "{module}" 9 "{out_name}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.emit_th(out_name, args) self.data += ".SH NAME\n" self.data += f"{args.type} {name} \\- {purpose}\n" diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index ca00695b47b3..74af7ae47aa4 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -13,7 +13,8 @@ import sys import re from pprint import pformat -from kdoc.kdoc_re import NestedMatch, KernRe +from kdoc.c_lex import CTokenizer, tokenizer_set_log +from kdoc.kdoc_re import KernRe from kdoc.kdoc_item import KdocItem # @@ -70,140 +71,9 @@ doc_begin_func = KernRe(str(doc_com) + # initial " * ' cache = False) # -# Here begins a long set of transformations to turn structure member prefixes -# and macro invocations into something we can parse and generate kdoc for. -# -struct_args_pattern = r'([^,)]+)' - -struct_xforms = [ - # Strip attributes - (KernRe(r"__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)", flags=re.I | re.S, cache=False), ' '), - (KernRe(r'\s*__aligned\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__counted_by\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__counted_by_(le|be)\s*\([^;]*\)', re.S), ' '), - (KernRe(r'\s*__packed\s*', re.S), ' '), - (KernRe(r'\s*CRYPTO_MINALIGN_ATTR', re.S), ' '), - (KernRe(r'\s*__private', re.S), ' '), - (KernRe(r'\s*__rcu', re.S), ' '), - (KernRe(r'\s*____cacheline_aligned_in_smp', re.S), ' '), - (KernRe(r'\s*____cacheline_aligned', re.S), ' '), - (KernRe(r'\s*__cacheline_group_(begin|end)\([^\)]+\);'), ''), - # - # Unwrap struct_group macros based on this definition: - # __struct_group(TAG, NAME, ATTRS, MEMBERS...) - # which has variants like: struct_group(NAME, MEMBERS...) - # Only MEMBERS arguments require documentation. - # - # Parsing them happens on two steps: - # - # 1. drop struct group arguments that aren't at MEMBERS, - # storing them as STRUCT_GROUP(MEMBERS) - # - # 2. remove STRUCT_GROUP() ancillary macro. - # - # The original logic used to remove STRUCT_GROUP() using an - # advanced regex: - # - # \bSTRUCT_GROUP(\(((?:(?>[^)(]+)|(?1))*)\))[^;]*; - # - # with two patterns that are incompatible with - # Python re module, as it has: - # - # - a recursive pattern: (?1) - # - an atomic grouping: (?>...) - # - # I tried a simpler version: but it didn't work either: - # \bSTRUCT_GROUP\(([^\)]+)\)[^;]*; - # - # As it doesn't properly match the end parenthesis on some cases. - # - # So, a better solution was crafted: there's now a NestedMatch - # class that ensures that delimiters after a search are properly - # matched. So, the implementation to drop STRUCT_GROUP() will be - # handled in separate. - # - (KernRe(r'\bstruct_group\s*\(([^,]*,)', re.S), r'STRUCT_GROUP('), - (KernRe(r'\bstruct_group_attr\s*\(([^,]*,){2}', re.S), r'STRUCT_GROUP('), - (KernRe(r'\bstruct_group_tagged\s*\(([^,]*),([^,]*),', re.S), r'struct \1 \2; STRUCT_GROUP('), - (KernRe(r'\b__struct_group\s*\(([^,]*,){3}', re.S), r'STRUCT_GROUP('), - # - # Replace macros - # - # TODO: use NestedMatch for FOO($1, $2, ...) matches - # - # it is better to also move those to the NestedMatch logic, - # to ensure that parentheses will be properly matched. - # - (KernRe(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), - r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), - (KernRe(r'DECLARE_PHY_INTERFACE_MASK\s*\(([^\)]+)\)', re.S), - r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), - (KernRe(r'DECLARE_BITMAP\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', - re.S), r'unsigned long \1[BITS_TO_LONGS(\2)]'), - (KernRe(r'DECLARE_HASHTABLE\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + r'\)', - re.S), r'unsigned long \1[1 << ((\2) - 1)]'), - (KernRe(r'DECLARE_KFIFO\s*\(' + struct_args_pattern + r',\s*' + struct_args_pattern + - r',\s*' + struct_args_pattern + r'\)', re.S), r'\2 *\1'), - (KernRe(r'DECLARE_KFIFO_PTR\s*\(' + struct_args_pattern + r',\s*' + - struct_args_pattern + r'\)', re.S), r'\2 *\1'), - (KernRe(r'(?:__)?DECLARE_FLEX_ARRAY\s*\(' + struct_args_pattern + r',\s*' + - struct_args_pattern + r'\)', re.S), r'\1 \2[]'), - (KernRe(r'DEFINE_DMA_UNMAP_ADDR\s*\(' + struct_args_pattern + r'\)', re.S), r'dma_addr_t \1'), - (KernRe(r'DEFINE_DMA_UNMAP_LEN\s*\(' + struct_args_pattern + r'\)', re.S), r'__u32 \1'), -] -# -# Regexes here are guaranteed to have the end delimiter matching -# the start delimiter. Yet, right now, only one replace group -# is allowed. -# -struct_nested_prefixes = [ - (re.compile(r'\bSTRUCT_GROUP\('), r'\1'), -] - -# -# Transforms for function prototypes -# -function_xforms = [ - (KernRe(r"^static +"), ""), - (KernRe(r"^extern +"), ""), - (KernRe(r"^asmlinkage +"), ""), - (KernRe(r"^inline +"), ""), - (KernRe(r"^__inline__ +"), ""), - (KernRe(r"^__inline +"), ""), - (KernRe(r"^__always_inline +"), ""), - (KernRe(r"^noinline +"), ""), - (KernRe(r"^__FORTIFY_INLINE +"), ""), - (KernRe(r"__init +"), ""), - (KernRe(r"__init_or_module +"), ""), - (KernRe(r"__exit +"), ""), - (KernRe(r"__deprecated +"), ""), - (KernRe(r"__flatten +"), ""), - (KernRe(r"__meminit +"), ""), - (KernRe(r"__must_check +"), ""), - (KernRe(r"__weak +"), ""), - (KernRe(r"__sched +"), ""), - (KernRe(r"_noprof"), ""), - (KernRe(r"__always_unused *"), ""), - (KernRe(r"__printf\s*\(\s*\d*\s*,\s*\d*\s*\) +"), ""), - (KernRe(r"__(?:re)?alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) +"), ""), - (KernRe(r"__diagnose_as\s*\(\s*\S+\s*(?:,\s*\d+\s*)*\) +"), ""), - (KernRe(r"DECL_BUCKET_PARAMS\s*\(\s*(\S+)\s*,\s*(\S+)\s*\)"), r"\1, \2"), - (KernRe(r"__attribute_const__ +"), ""), - (KernRe(r"__attribute__\s*\(\((?:[\w\s]+(?:\([^)]*\))?\s*,?)+\)\)\s+"), ""), -] - -# # Ancillary functions # -def apply_transforms(xforms, text): - """ - Apply a set of transforms to a block of text. - """ - for search, subst in xforms: - text = search.sub(subst, text) - return text - multi_space = KernRe(r'\s\s+') def trim_whitespace(s): """ @@ -215,15 +85,9 @@ def trim_private_members(text): """ Remove ``struct``/``enum`` members that have been marked "private". """ - # First look for a "public:" block that ends a private region, then - # handle the "private until the end" case. - # - text = KernRe(r'/\*\s*private:.*?/\*\s*public:.*?\*/', flags=re.S).sub('', text) - text = KernRe(r'/\*\s*private:.*', flags=re.S).sub('', text) - # - # We needed the comments to do the above, but now we can take them out. - # - return KernRe(r'\s*/\*.*?\*/\s*', flags=re.S).sub('', text).strip() + + tokens = CTokenizer(text) + return str(tokens) class state: """ @@ -276,7 +140,7 @@ class KernelEntry: self.parametertypes = {} self.parameterdesc_start_lines = {} - self.section_start_lines = {} + self.sections_start_lines = {} self.sections = {} self.anon_struct_union = False @@ -356,7 +220,7 @@ class KernelEntry: self.sections[name] += '\n' + contents else: self.sections[name] = contents - self.section_start_lines[name] = self.new_start_line + self.sections_start_lines[name] = self.new_start_line self.new_start_line = 0 # self.config.log.debug("Section: %s : %s", name, pformat(vars(self))) @@ -382,11 +246,15 @@ class KernelDoc: #: String to write when a parameter is not described. undescribed = "-- undescribed --" - def __init__(self, config, fname): + def __init__(self, config, fname, xforms, store_src=False): """Initialize internal variables""" self.fname = fname self.config = config + self.xforms = xforms + self.store_src = store_src + + tokenizer_set_log(self.config.log, f"{self.fname}: CMatch: ") # Initial state for the state machines self.state = state.NORMAL @@ -449,7 +317,7 @@ class KernelDoc: for section in ["Description", "Return"]: if section in sections and not sections[section].rstrip(): del sections[section] - item.set_sections(sections, self.entry.section_start_lines) + item.set_sections(sections, self.entry.sections_start_lines) item.set_params(self.entry.parameterlist, self.entry.parameterdescs, self.entry.parametertypes, self.entry.parameterdesc_start_lines) @@ -849,13 +717,15 @@ class KernelDoc: return declaration - def dump_struct(self, ln, proto): + def dump_struct(self, ln, proto, source): """ Store an entry for a ``struct`` or ``union`` """ # # Do the basic parse to get the pieces of the declaration. # + source = source + proto = trim_private_members(proto) struct_parts = self.split_struct_proto(proto) if not struct_parts: self.emit_msg(ln, f"{proto} error: Cannot parse struct or union!") @@ -869,12 +739,8 @@ class KernelDoc: # # Go through the list of members applying all of our transformations. # - members = trim_private_members(members) - members = apply_transforms(struct_xforms, members) + members = self.xforms.apply("struct", members) - nested = NestedMatch() - for search, sub in struct_nested_prefixes: - members = nested.sub(search, sub, members) # # Deal with embedded struct and union members, and drop enums entirely. # @@ -888,10 +754,11 @@ class KernelDoc: declaration_name) self.check_sections(ln, declaration_name, decl_type) self.output_declaration(decl_type, declaration_name, + source=source, definition=self.format_struct_decl(declaration), purpose=self.entry.declaration_purpose) - def dump_enum(self, ln, proto): + def dump_enum(self, ln, proto, source): """ Store an ``enum`` inside self.entries array. """ @@ -899,6 +766,8 @@ class KernelDoc: # Strip preprocessor directives. Note that this depends on the # trailing semicolon we added in process_proto_type(). # + source = source + proto = trim_private_members(proto) proto = KernRe(r'#\s*((define|ifdef|if)\s+|endif)[^;]*;', flags=re.S).sub('', proto) # # Parse out the name and members of the enum. Typedef form first. @@ -906,7 +775,7 @@ class KernelDoc: r = KernRe(r'typedef\s+enum\s*\{(.*)\}\s*(\w*)\s*;') if r.search(proto): declaration_name = r.group(2) - members = trim_private_members(r.group(1)) + members = r.group(1) # # Failing that, look for a straight enum # @@ -914,7 +783,7 @@ class KernelDoc: r = KernRe(r'enum\s+(\w*)\s*\{(.*)\}') if r.match(proto): declaration_name = r.group(1) - members = trim_private_members(r.group(2)) + members = r.group(2) # # OK, this isn't going to work. # @@ -943,9 +812,10 @@ class KernelDoc: member_set = set() members = KernRe(r'\([^;)]*\)').sub('', members) for arg in members.split(','): - if not arg: - continue arg = KernRe(r'^\s*(\w+).*').sub(r'\1', arg) + if not arg.strip(): + continue + self.entry.parameterlist.append(arg) if arg not in self.entry.parameterdescs: self.entry.parameterdescs[arg] = self.undescribed @@ -961,29 +831,23 @@ class KernelDoc: f"Excess enum value '@{k}' description in '{declaration_name}'") self.output_declaration('enum', declaration_name, + source=source, purpose=self.entry.declaration_purpose) - def dump_var(self, ln, proto): + def dump_var(self, ln, proto, source): """ Store variables that are part of kAPI. """ VAR_ATTRIBS = [ "extern", + "const", ] - OPTIONAL_VAR_ATTR = "^(?:" + "|".join(VAR_ATTRIBS) + ")?" - - sub_prefixes = [ - (KernRe(r"__read_mostly"), ""), - (KernRe(r"__ro_after_init"), ""), - (KernRe(r"(?://.*)$"), ""), - (KernRe(r"(?:/\*.*\*/)"), ""), - (KernRe(r";$"), ""), - (KernRe(r"=.*"), ""), - ] + OPTIONAL_VAR_ATTR = r"^(?:\b(?:" +"|".join(VAR_ATTRIBS) +r")\b\s*)*" # # Store the full prototype before modifying it # + source = source full_proto = proto declaration_name = None @@ -1004,8 +868,7 @@ class KernelDoc: # Drop comments and macros to have a pure C prototype # if not declaration_name: - for r, sub in sub_prefixes: - proto = r.sub(sub, proto) + proto = self.xforms.apply("var", proto) proto = proto.rstrip() @@ -1015,17 +878,17 @@ class KernelDoc: default_val = None - r= KernRe(OPTIONAL_VAR_ATTR + r"\w.*\s+(?:\*+)?([\w_]+)\s*[\d\]\[]*\s*(=.*)?") + r= KernRe(OPTIONAL_VAR_ATTR + r"\s*[\w_\s]*\s+(?:\*+)?([\w_]+)\s*[\d\]\[]*\s*(=.*)?") if r.match(proto): if not declaration_name: declaration_name = r.group(1) default_val = r.group(2) else: - r= KernRe(OPTIONAL_VAR_ATTR + r"(?:\w.*)?\s+(?:\*+)?(?:[\w_]+)\s*[\d\]\[]*\s*(=.*)?") - if r.match(proto): - default_val = r.group(1) + r= KernRe(OPTIONAL_VAR_ATTR + r"(?:[\w_\s]*)?\s+(?:\*+)?(?:[\w_]+)\s*[\d\]\[]*\s*(=.*)?") + if r.match(proto): + default_val = r.group(1) if not declaration_name: self.emit_msg(ln,f"{proto}: can't parse variable") return @@ -1034,39 +897,38 @@ class KernelDoc: default_val = default_val.lstrip("=").strip() self.output_declaration("var", declaration_name, + source=source, full_proto=full_proto, default_val=default_val, purpose=self.entry.declaration_purpose) - def dump_declaration(self, ln, prototype): + def dump_declaration(self, ln, prototype, source): """ Store a data declaration inside self.entries array. """ if self.entry.decl_type == "enum": - self.dump_enum(ln, prototype) + self.dump_enum(ln, prototype, source) elif self.entry.decl_type == "typedef": - self.dump_typedef(ln, prototype) + self.dump_typedef(ln, prototype, source) elif self.entry.decl_type in ["union", "struct"]: - self.dump_struct(ln, prototype) + self.dump_struct(ln, prototype, source) elif self.entry.decl_type == "var": - self.dump_var(ln, prototype) + self.dump_var(ln, prototype, source) else: # This would be a bug self.emit_message(ln, f'Unknown declaration type: {self.entry.decl_type}') - def dump_function(self, ln, prototype): + def dump_function(self, ln, prototype, source): """ Store a function or function macro inside self.entries array. """ + source = source found = func_macro = False return_type = '' decl_type = 'function' - # - # Apply the initial transformations. - # - prototype = apply_transforms(function_xforms, prototype) + # # If we have a macro, remove the "#define" at the front. # @@ -1085,6 +947,11 @@ class KernelDoc: declaration_name = r.group(1) func_macro = True found = True + else: + # + # Apply the initial transformations. + # + prototype = self.xforms.apply("func", prototype) # Yes, this truly is vile. We are looking for: # 1. Return type (may be nothing if we're looking at a macro) @@ -1150,13 +1017,14 @@ class KernelDoc: # Store the result. # self.output_declaration(decl_type, declaration_name, + source=source, typedef=('typedef' in return_type), functiontype=return_type, purpose=self.entry.declaration_purpose, func_macro=func_macro) - def dump_typedef(self, ln, proto): + def dump_typedef(self, ln, proto, source): """ Store a ``typedef`` inside self.entries array. """ @@ -1167,6 +1035,8 @@ class KernelDoc: typedef_ident = r'\*?\s*(\w\S+)\s*' typedef_args = r'\s*\((.*)\);' + source = source + typedef1 = KernRe(typedef_type + r'\(' + typedef_ident + r'\)' + typedef_args) typedef2 = KernRe(typedef_type + typedef_ident + typedef_args) @@ -1187,6 +1057,7 @@ class KernelDoc: self.create_parameter_list(ln, 'function', args, ',', declaration_name) self.output_declaration('function', declaration_name, + source=source, typedef=True, functiontype=return_type, purpose=self.entry.declaration_purpose) @@ -1204,6 +1075,7 @@ class KernelDoc: return self.output_declaration('typedef', declaration_name, + source=source, purpose=self.entry.declaration_purpose) return @@ -1241,7 +1113,7 @@ class KernelDoc: function_set.add(symbol) return True - def process_normal(self, ln, line): + def process_normal(self, ln, line, source): """ STATE_NORMAL: looking for the ``/**`` to begin everything. """ @@ -1255,7 +1127,7 @@ class KernelDoc: # next line is always the function name self.state = state.NAME - def process_name(self, ln, line): + def process_name(self, ln, line, source): """ STATE_NAME: Looking for the "name - description" line """ @@ -1388,7 +1260,7 @@ class KernelDoc: return False - def process_decl(self, ln, line): + def process_decl(self, ln, line, source): """ STATE_DECLARATION: We've seen the beginning of a declaration. """ @@ -1417,7 +1289,7 @@ class KernelDoc: self.emit_msg(ln, f"bad line: {line}") - def process_special(self, ln, line): + def process_special(self, ln, line, source): """ STATE_SPECIAL_SECTION: a section ending with a blank line. """ @@ -1468,7 +1340,7 @@ class KernelDoc: # Unknown line, ignore self.emit_msg(ln, f"bad line: {line}") - def process_body(self, ln, line): + def process_body(self, ln, line, source): """ STATE_BODY: the bulk of a kerneldoc comment. """ @@ -1482,7 +1354,7 @@ class KernelDoc: # Unknown line, ignore self.emit_msg(ln, f"bad line: {line}") - def process_inline_name(self, ln, line): + def process_inline_name(self, ln, line, source): """STATE_INLINE_NAME: beginning of docbook comments within a prototype.""" if doc_inline_sect.search(line): @@ -1495,9 +1367,15 @@ class KernelDoc: elif doc_content.search(line): self.emit_msg(ln, f"Incorrect use of kernel-doc format: {line}") self.state = state.PROTO + + # + # Don't let it add partial comments at the code, as breaks the + # logic meant to remove comments from prototypes. + # + self.process_proto_type(ln, "/**\n" + line, source) # else ... ?? - def process_inline_text(self, ln, line): + def process_inline_text(self, ln, line, source): """STATE_INLINE_TEXT: docbook comments within a prototype.""" if doc_inline_end.search(line): @@ -1583,7 +1461,7 @@ class KernelDoc: return proto - def process_proto_function(self, ln, line): + def process_proto_function(self, ln, line, source): """Ancillary routine to process a function prototype.""" # strip C99-style comments to end of line @@ -1625,10 +1503,10 @@ class KernelDoc: # # ... and we're done # - self.dump_function(ln, self.entry.prototype) + self.dump_function(ln, self.entry.prototype, source) self.reset_state(ln) - def process_proto_type(self, ln, line): + def process_proto_type(self, ln, line, source): """ Ancillary routine to process a type. """ @@ -1658,7 +1536,7 @@ class KernelDoc: elif chunk == '}': self.entry.brcount -= 1 elif chunk == ';' and self.entry.brcount <= 0: - self.dump_declaration(ln, self.entry.prototype) + self.dump_declaration(ln, self.entry.prototype, source) self.reset_state(ln) return # @@ -1667,7 +1545,7 @@ class KernelDoc: # self.entry.prototype += ' ' - def process_proto(self, ln, line): + def process_proto(self, ln, line, source): """STATE_PROTO: reading a function/whatever prototype.""" if doc_inline_oneline.search(line): @@ -1679,17 +1557,18 @@ class KernelDoc: self.state = state.INLINE_NAME elif self.entry.decl_type == 'function': - self.process_proto_function(ln, line) + self.process_proto_function(ln, line, source) else: - self.process_proto_type(ln, line) + self.process_proto_type(ln, line, source) - def process_docblock(self, ln, line): + def process_docblock(self, ln, line, source): """STATE_DOCBLOCK: within a ``DOC:`` block.""" if doc_end.search(line): self.dump_section() - self.output_declaration("doc", self.entry.identifier) + self.output_declaration("doc", self.entry.identifier, + source=source) self.reset_state(ln) elif doc_content.search(line): @@ -1740,6 +1619,8 @@ class KernelDoc: prev = "" prev_ln = None export_table = set() + self.state = state.NORMAL + source = "" try: with open(self.fname, "r", encoding="utf8", @@ -1766,6 +1647,12 @@ class KernelDoc: ln, state.name[self.state], line) + if self.store_src: + if source and self.state == state.NORMAL: + source = "" + elif self.state != state.NORMAL: + source += line + "\n" + # This is an optimization over the original script. # There, when export_file was used for the same file, # it was read twice. Here, we use the already-existing @@ -1773,8 +1660,11 @@ class KernelDoc: # if (self.state != state.NORMAL) or \ not self.process_export(export_table, line): + prev_state = self.state # Hand this line to the appropriate state handler - self.state_actions[self.state](self, ln, line) + self.state_actions[self.state](self, ln, line, source) + if prev_state == state.NORMAL and self.state != state.NORMAL: + source += line + "\n" self.emit_unused_warnings() diff --git a/tools/lib/python/kdoc/kdoc_re.py b/tools/lib/python/kdoc/kdoc_re.py index 0bf9e01cdc57..28292efe25a2 100644 --- a/tools/lib/python/kdoc/kdoc_re.py +++ b/tools/lib/python/kdoc/kdoc_re.py @@ -52,7 +52,33 @@ class KernRe: return self.regex.pattern def __repr__(self): - return f're.compile("{self.regex.pattern}")' + """ + Returns a displayable version of the class init. + """ + + flag_map = { + re.IGNORECASE: "re.I", + re.MULTILINE: "re.M", + re.DOTALL: "re.S", + re.VERBOSE: "re.X", + } + + flags = [] + for flag, name in flag_map.items(): + if self.regex.flags & flag: + flags.append(name) + + flags_name = " | ".join(flags) + + max_len = 60 + pattern = "" + for pos in range(0, len(self.regex.pattern), max_len): + pattern += '"' + self.regex.pattern[pos:max_len + pos] + '" ' + + if flags_name: + return f'KernRe({pattern}, {flags_name})' + else: + return f'KernRe({pattern})' def __add__(self, other): """ @@ -78,6 +104,13 @@ class KernRe: self.last_match = self.regex.search(string) return self.last_match + def finditer(self, string): + """ + Alias to re.finditer. + """ + + return self.regex.finditer(string) + def findall(self, string): """ Alias to re.findall. @@ -106,173 +139,9 @@ class KernRe: return self.last_match.group(num) - -class NestedMatch: - """ - Finding nested delimiters is hard with regular expressions. It is - even harder on Python with its normal re module, as there are several - advanced regular expressions that are missing. - - This is the case of this pattern:: - - '\\bSTRUCT_GROUP(\\(((?:(?>[^)(]+)|(?1))*)\\))[^;]*;' - - which is used to properly match open/close parentheses of the - string search STRUCT_GROUP(), - - Add a class that counts pairs of delimiters, using it to match and - replace nested expressions. - - The original approach was suggested by: - - https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex - - Although I re-implemented it to make it more generic and match 3 types - of delimiters. The logic checks if delimiters are paired. If not, it - will ignore the search string. - """ - - # TODO: make NestedMatch handle multiple match groups - # - # Right now, regular expressions to match it are defined only up to - # the start delimiter, e.g.: - # - # \bSTRUCT_GROUP\( - # - # is similar to: STRUCT_GROUP\((.*)\) - # except that the content inside the match group is delimiter-aligned. - # - # The content inside parentheses is converted into a single replace - # group (e.g. r`\1'). - # - # It would be nice to change such definition to support multiple - # match groups, allowing a regex equivalent to: - # - # FOO\((.*), (.*), (.*)\) - # - # it is probably easier to define it not as a regular expression, but - # with some lexical definition like: - # - # FOO(arg1, arg2, arg3) - - DELIMITER_PAIRS = { - '{': '}', - '(': ')', - '[': ']', - } - - RE_DELIM = re.compile(r'[\{\}\[\]\(\)]') - - def _search(self, regex, line): - """ - Finds paired blocks for a regex that ends with a delimiter. - - The suggestion of using finditer to match pairs came from: - https://stackoverflow.com/questions/5454322/python-how-to-match-nested-parentheses-with-regex - but I ended using a different implementation to align all three types - of delimiters and seek for an initial regular expression. - - The algorithm seeks for open/close paired delimiters and places them - into a stack, yielding a start/stop position of each match when the - stack is zeroed. - - The algorithm should work fine for properly paired lines, but will - silently ignore end delimiters that precede a start delimiter. - This should be OK for kernel-doc parser, as unaligned delimiters - would cause compilation errors. So, we don't need to raise exceptions - to cover such issues. - """ - - stack = [] - - for match_re in regex.finditer(line): - start = match_re.start() - offset = match_re.end() - - d = line[offset - 1] - if d not in self.DELIMITER_PAIRS: - continue - - end = self.DELIMITER_PAIRS[d] - stack.append(end) - - for match in self.RE_DELIM.finditer(line[offset:]): - pos = match.start() + offset - - d = line[pos] - - if d in self.DELIMITER_PAIRS: - end = self.DELIMITER_PAIRS[d] - - stack.append(end) - continue - - # Does the end delimiter match what is expected? - if stack and d == stack[-1]: - stack.pop() - - if not stack: - yield start, offset, pos + 1 - break - - def search(self, regex, line): + def groups(self): """ - This is similar to re.search: - - It matches a regex that it is followed by a delimiter, - returning occurrences only if all delimiters are paired. + Returns the group results of the last match """ - for t in self._search(regex, line): - - yield line[t[0]:t[2]] - - def sub(self, regex, sub, line, count=0): - r""" - This is similar to re.sub: - - It matches a regex that it is followed by a delimiter, - replacing occurrences only if all delimiters are paired. - - if the sub argument contains:: - - r'\1' - - it will work just like re: it places there the matched paired data - with the delimiter stripped. - - If count is different than zero, it will replace at most count - items. - """ - out = "" - - cur_pos = 0 - n = 0 - - for start, end, pos in self._search(regex, line): - out += line[cur_pos:start] - - # Value, ignoring start/end delimiters - value = line[end:pos - 1] - - # replaces \1 at the sub string, if \1 is used there - new_sub = sub - new_sub = new_sub.replace(r'\1', value) - - out += new_sub - - # Drop end ';' if any - if line[pos] == ';': - pos += 1 - - cur_pos = pos - n += 1 - - if count and count >= n: - break - - # Append the remaining string - l = len(line) - out += line[cur_pos:l] - - return out + return self.last_match.groups() diff --git a/tools/lib/python/kdoc/kdoc_yaml_file.py b/tools/lib/python/kdoc/kdoc_yaml_file.py new file mode 100644 index 000000000000..0be020d50df0 --- /dev/null +++ b/tools/lib/python/kdoc/kdoc_yaml_file.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2026: Mauro Carvalho Chehab <mchehab@kernel.org>. + +import os + +from kdoc.kdoc_output import ManFormat, RestFormat + + +class KDocTestFile(): + """ + Handles the logic needed to store kernel‑doc output inside a YAML file. + Useful for unit tests and regression tests. + """ + + def __init__(self, config, yaml_file, yaml_content): + # + # Bail out early if yaml is not available + # + try: + import yaml + except ImportError: + sys.exit("Warning: yaml package not available. Aborting it.") + + self.config = config + self.test_file = os.path.expanduser(yaml_file) + self.yaml_content = yaml_content + self.test_names = set() + + self.tests = [] + + out_dir = os.path.dirname(self.test_file) + if out_dir and not os.path.isdir(out_dir): + sys.exit(f"Directory {out_dir} doesn't exist.") + + self.out_style = [] + + if "man" in self.yaml_content: + out_style = ManFormat() + out_style.set_config(self.config) + + self.out_style.append(out_style) + + if "rst" in self.yaml_content: + out_style = RestFormat() + out_style.set_config(self.config) + + self.out_style.append(out_style) + + def set_filter(self, export, internal, symbol, nosymbol, + function_table, enable_lineno, no_doc_sections): + """ + Set filters at the output classes. + """ + for out_style in self.out_style: + out_style.set_filter(export, internal, symbol, + nosymbol, function_table, + enable_lineno, no_doc_sections) + + @staticmethod + def get_kdoc_item(arg, start_line=1): + + d = vars(arg) + + declaration_start_line = d.get("declaration_start_line") + if not declaration_start_line: + return d + + d["declaration_start_line"] = start_line + + parameterdesc_start_lines = d.get("parameterdesc_start_lines") + if parameterdesc_start_lines: + for key in parameterdesc_start_lines: + ln = parameterdesc_start_lines[key] + ln += start_line - declaration_start_line + + parameterdesc_start_lines[key] = ln + + sections_start_lines = d.get("sections_start_lines") + if sections_start_lines: + for key in sections_start_lines: + ln = sections_start_lines[key] + ln += start_line - declaration_start_line + + sections_start_lines[key] = ln + + return d + + def output_symbols(self, fname, symbols): + """ + Store source, symbols and output strings at self.tests. + """ + + # + # KdocItem needs to be converted into dicts + # + kdoc_item = [] + expected = [] + + # + # Source code didn't produce any symbol + # + if not symbols: + return + + expected_dict = {} + start_line=1 + + for arg in symbols: + source = arg.get("source", "") + + if arg and "KdocItem" in self.yaml_content: + msg = self.get_kdoc_item(arg) + + other_stuff = msg.get("other_stuff", {}) + if "source" in other_stuff: + del other_stuff["source"] + + expected_dict["kdoc_item"] = msg + + base_name = arg.name + if not base_name: + base_name = fname + base_name = base_name.lower().replace(".", "_").replace("/", "_") + + + # Don't add duplicated names + i = 0 + name = base_name + while name in self.test_names: + i += 1 + name = f"{base_name}_{i:03d}" + + self.test_names.add(name) + + for out_style in self.out_style: + if isinstance(out_style, ManFormat): + key = "man" + else: + key = "rst" + + expected_dict[key]= out_style.output_symbols(fname, [arg]).strip() + + test = { + "name": name, + "description": f"{fname} line {arg.declaration_start_line}", + "fname": fname, + "source": source, + "expected": [expected_dict] + } + + self.tests.append(test) + + expected_dict = {} + + def write(self): + """ + Output the content of self.tests to self.test_file. + """ + import yaml + + # Helper function to better handle multilines + def str_presenter(dumper, data): + if "\n" in data: + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + + return dumper.represent_scalar("tag:yaml.org,2002:str", data) + + # Register the representer + yaml.add_representer(str, str_presenter) + + data = {"tests": self.tests} + + with open(self.test_file, "w", encoding="utf-8") as fp: + yaml.dump(data, fp, + sort_keys=False, width=120, indent=2, + default_flow_style=False, allow_unicode=True, + explicit_start=False, explicit_end=False) diff --git a/tools/lib/python/kdoc/xforms_lists.py b/tools/lib/python/kdoc/xforms_lists.py new file mode 100644 index 000000000000..f6ea9efb11ae --- /dev/null +++ b/tools/lib/python/kdoc/xforms_lists.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2026: Mauro Carvalho Chehab <mchehab@kernel.org>. + +import re + +from kdoc.kdoc_re import KernRe +from kdoc.c_lex import CMatch, CTokenizer + +struct_args_pattern = r"([^,)]+)" + + +class CTransforms: + """ + Data class containing a long set of transformations to turn + structure member prefixes, and macro invocations and variables + into something we can parse and generate kdoc for. + """ + + # + # NOTE: + # Due to performance reasons, place CMatch rules before KernRe, + # as this avoids running the C parser every time. + # + + #: Transforms for structs and unions. + struct_xforms = [ + (CMatch("__attribute__"), ""), + (CMatch("__aligned"), ""), + (CMatch("__counted_by"), ""), + (CMatch("__counted_by_(le|be)"), ""), + (CMatch("__guarded_by"), ""), + (CMatch("__pt_guarded_by"), ""), + (CMatch("__packed"), ""), + (CMatch("CRYPTO_MINALIGN_ATTR"), ""), + (CMatch("__private"), ""), + (CMatch("__rcu"), ""), + (CMatch("____cacheline_aligned_in_smp"), ""), + (CMatch("____cacheline_aligned"), ""), + (CMatch("__cacheline_group_(?:begin|end)"), ""), + (CMatch("__ETHTOOL_DECLARE_LINK_MODE_MASK"), r"DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)"), + (CMatch("DECLARE_PHY_INTERFACE_MASK",),r"DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)"), + (CMatch("DECLARE_BITMAP"), r"unsigned long \1[BITS_TO_LONGS(\2)]"), + (CMatch("DECLARE_HASHTABLE"), r"unsigned long \1[1 << ((\2) - 1)]"), + (CMatch("DECLARE_KFIFO"), r"\2 *\1"), + (CMatch("DECLARE_KFIFO_PTR"), r"\2 *\1"), + (CMatch("(?:__)?DECLARE_FLEX_ARRAY"), r"\1 \2[]"), + (CMatch("DEFINE_DMA_UNMAP_ADDR"), r"dma_addr_t \1"), + (CMatch("DEFINE_DMA_UNMAP_LEN"), r"__u32 \1"), + (CMatch("VIRTIO_DECLARE_FEATURES"), r"union { u64 \1; u64 \1_array[VIRTIO_FEATURES_U64S]; }"), + (CMatch("__cond_acquires"), ""), + (CMatch("__cond_releases"), ""), + (CMatch("__acquires"), ""), + (CMatch("__releases"), ""), + (CMatch("__must_hold"), ""), + (CMatch("__must_not_hold"), ""), + (CMatch("__must_hold_shared"), ""), + (CMatch("__cond_acquires_shared"), ""), + (CMatch("__acquires_shared"), ""), + (CMatch("__releases_shared"), ""), + (CMatch("__attribute__"), ""), + + # + # Macro __struct_group() creates an union with an anonymous + # and a non-anonymous struct, depending on the parameters. We only + # need one of those at kernel-doc, as we won't be documenting the same + # members twice. + # + (CMatch("struct_group"), r"struct { \2+ };"), + (CMatch("struct_group_attr"), r"struct { \3+ };"), + (CMatch("struct_group_tagged"), r"struct { \3+ };"), + (CMatch("__struct_group"), r"struct { \4+ };"), + ] + + #: Transforms for function prototypes. + function_xforms = [ + (CMatch("static"), ""), + (CMatch("extern"), ""), + (CMatch("asmlinkage"), ""), + (CMatch("inline"), ""), + (CMatch("__inline__"), ""), + (CMatch("__inline"), ""), + (CMatch("__always_inline"), ""), + (CMatch("noinline"), ""), + (CMatch("__FORTIFY_INLINE"), ""), + (CMatch("__init"), ""), + (CMatch("__init_or_module"), ""), + (CMatch("__exit"), ""), + (CMatch("__deprecated"), ""), + (CMatch("__flatten"), ""), + (CMatch("__meminit"), ""), + (CMatch("__must_check"), ""), + (CMatch("__weak"), ""), + (CMatch("__sched"), ""), + (CMatch("__always_unused"), ""), + (CMatch("__printf"), ""), + (CMatch("__(?:re)?alloc_size"), ""), + (CMatch("__diagnose_as"), ""), + (CMatch("DECL_BUCKET_PARAMS"), r"\1, \2"), + (CMatch("__no_context_analysis"), ""), + (CMatch("__attribute_const__"), ""), + (CMatch("__attribute__"), ""), + + # + # HACK: this is similar to process_export() hack. It is meant to + # drop _noproof from function name. See for instance: + # ahash_request_alloc kernel-doc declaration at include/crypto/hash.h. + # + (KernRe("_noprof"), ""), + ] + + #: Transforms for variable prototypes. + var_xforms = [ + (CMatch("__read_mostly"), ""), + (CMatch("__ro_after_init"), ""), + (CMatch("__guarded_by"), ""), + (CMatch("__pt_guarded_by"), ""), + (CMatch("LIST_HEAD"), r"struct list_head \1"), + + (KernRe(r"(?://.*)$"), ""), + (KernRe(r"(?:/\*.*\*/)"), ""), + (KernRe(r";$"), ""), + ] + + #: Transforms main dictionary used at apply_transforms(). + xforms = { + "struct": struct_xforms, + "func": function_xforms, + "var": var_xforms, + } + + def apply(self, xforms_type, source): + """ + Apply a set of transforms to a block of source. + + As tokenizer is used here, this function also remove comments + at the end. + """ + if xforms_type not in self.xforms: + return source + + if isinstance(source, str): + source = CTokenizer(source) + + for search, subst in self.xforms[xforms_type]: + # + # KernRe only accept strings. + # + if isinstance(search, KernRe): + source = str(source) + + source = search.sub(subst, source) + return str(source) diff --git a/tools/lib/python/unittest_helper.py b/tools/lib/python/unittest_helper.py new file mode 100755 index 000000000000..f3cba5120401 --- /dev/null +++ b/tools/lib/python/unittest_helper.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2025-2026: Mauro Carvalho Chehab <mchehab@kernel.org>. +# +# pylint: disable=C0103,R0912,R0914,E1101 + +""" +Provides helper functions and classes execute python unit tests. + +Those help functions provide a nice colored output summary of each +executed test and, when a test fails, it shows the different in diff +format when running in verbose mode, like:: + + $ tools/unittests/nested_match.py -v + ... + Traceback (most recent call last): + File "/new_devel/docs/tools/unittests/nested_match.py", line 69, in test_count_limit + self.assertEqual(replaced, "bar(a); bar(b); foo(c)") + ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + AssertionError: 'bar(a) foo(b); foo(c)' != 'bar(a); bar(b); foo(c)' + - bar(a) foo(b); foo(c) + ? ^^^^ + + bar(a); bar(b); foo(c) + ? ^^^^^ + ... + +It also allows filtering what tests will be executed via ``-k`` parameter. + +Typical usage is to do:: + + from unittest_helper import run_unittest + ... + + if __name__ == "__main__": + run_unittest(__file__) + +If passing arguments is needed, on a more complex scenario, it can be +used like on this example:: + + from unittest_helper import TestUnits, run_unittest + ... + env = {'sudo': ""} + ... + if __name__ == "__main__": + runner = TestUnits() + base_parser = runner.parse_args() + base_parser.add_argument('--sudo', action='store_true', + help='Enable tests requiring sudo privileges') + + args = base_parser.parse_args() + + # Update module-level flag + if args.sudo: + env['sudo'] = "1" + + # Run tests with customized arguments + runner.run(__file__, parser=base_parser, args=args, env=env) +""" + +import argparse +import atexit +import os +import re +import unittest +import sys + +from unittest.mock import patch + + +class Summary(unittest.TestResult): + """ + Overrides ``unittest.TestResult`` class to provide a nice colored + summary. When in verbose mode, displays actual/expected difference in + unified diff format. + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + #: Dictionary to store organized test results. + self.test_results = {} + + #: max length of the test names. + self.max_name_length = 0 + + def startTest(self, test): + super().startTest(test) + test_id = test.id() + parts = test_id.split(".") + + # Extract module, class, and method names + if len(parts) >= 3: + module_name = parts[-3] + else: + module_name = "" + if len(parts) >= 2: + class_name = parts[-2] + else: + class_name = "" + + method_name = parts[-1] + + # Build the hierarchical structure + if module_name not in self.test_results: + self.test_results[module_name] = {} + + if class_name not in self.test_results[module_name]: + self.test_results[module_name][class_name] = [] + + # Track maximum test name length for alignment + display_name = f"{method_name}:" + + self.max_name_length = max(len(display_name), self.max_name_length) + + def _record_test(self, test, status): + test_id = test.id() + parts = test_id.split(".") + if len(parts) >= 3: + module_name = parts[-3] + else: + module_name = "" + if len(parts) >= 2: + class_name = parts[-2] + else: + class_name = "" + method_name = parts[-1] + self.test_results[module_name][class_name].append((method_name, status)) + + def addSuccess(self, test): + super().addSuccess(test) + self._record_test(test, "OK") + + def addFailure(self, test, err): + super().addFailure(test, err) + self._record_test(test, "FAIL") + + def addError(self, test, err): + super().addError(test, err) + self._record_test(test, "ERROR") + + def addSkip(self, test, reason): + super().addSkip(test, reason) + self._record_test(test, f"SKIP ({reason})") + + def printResults(self, verbose): + """ + Print results using colors if tty. + """ + # Check for ANSI color support + use_color = sys.stdout.isatty() + COLORS = { + "OK": "\033[32m", # Green + "FAIL": "\033[31m", # Red + "SKIP": "\033[1;33m", # Yellow + "PARTIAL": "\033[33m", # Orange + "EXPECTED_FAIL": "\033[36m", # Cyan + "reset": "\033[0m", # Reset to default terminal color + } + if not use_color: + for c in COLORS: + COLORS[c] = "" + + # Calculate maximum test name length + if not self.test_results: + return + try: + lengths = [] + for module in self.test_results.values(): + for tests in module.values(): + for test_name, _ in tests: + lengths.append(len(test_name) + 1) # +1 for colon + max_length = max(lengths) + 2 # Additional padding + except ValueError: + sys.exit("Test list is empty") + + # Print results + for module_name, classes in self.test_results.items(): + if verbose: + print(f"{module_name}:") + for class_name, tests in classes.items(): + if verbose: + print(f" {class_name}:") + for test_name, status in tests: + if not verbose and status in [ "OK", "EXPECTED_FAIL" ]: + continue + + # Get base status without reason for SKIP + if status.startswith("SKIP"): + status_code = status.split()[0] + else: + status_code = status + color = COLORS.get(status_code, "") + print( + f" {test_name + ':':<{max_length}}{color}{status}{COLORS['reset']}" + ) + if verbose: + print() + + # Print summary + print(f"\nRan {self.testsRun} tests", end="") + if hasattr(self, "timeTaken"): + print(f" in {self.timeTaken:.3f}s", end="") + print() + + if not self.wasSuccessful(): + print(f"\n{COLORS['FAIL']}FAILED (", end="") + failures = getattr(self, "failures", []) + errors = getattr(self, "errors", []) + if failures: + print(f"failures={len(failures)}", end="") + if errors: + if failures: + print(", ", end="") + print(f"errors={len(errors)}", end="") + print(f"){COLORS['reset']}") + + +def flatten_suite(suite): + """Flatten test suite hierarchy.""" + tests = [] + for item in suite: + if isinstance(item, unittest.TestSuite): + tests.extend(flatten_suite(item)) + else: + tests.append(item) + return tests + + +class TestUnits: + """ + Helper class to set verbosity level. + + This class discover test files, import its unittest classes and + executes the test on it. + """ + def parse_args(self): + """Returns a parser for command line arguments.""" + parser = argparse.ArgumentParser(description="Test runner with regex filtering") + parser.add_argument("-v", "--verbose", action="count", default=1) + parser.add_argument("-q", "--quiet", action="store_true") + parser.add_argument("-f", "--failfast", action="store_true") + parser.add_argument("-k", "--keyword", + help="Regex pattern to filter test methods") + return parser + + def run(self, caller_file=None, pattern=None, + suite=None, parser=None, args=None, env=None): + """ + Execute all tests from the unity test file. + + It contains several optional parameters: + + ``caller_file``: + - name of the file that contains test. + + typical usage is to place __file__ at the caller test, e.g.:: + + if __name__ == "__main__": + TestUnits().run(__file__) + + ``pattern``: + - optional pattern to match multiple file names. Defaults + to basename of ``caller_file``. + + ``suite``: + - an unittest suite initialized by the caller using + ``unittest.TestLoader().discover()``. + + ``parser``: + - an argparse parser. If not defined, this helper will create + one. + + ``args``: + - an ``argparse.Namespace`` data filled by the caller. + + ``env``: + - environment variables that will be passed to the test suite + + At least ``caller_file`` or ``suite`` must be used, otherwise a + ``TypeError`` will be raised. + """ + if not args: + if not parser: + parser = self.parse_args() + args = parser.parse_args() + + if not caller_file and not suite: + raise TypeError("Either caller_file or suite is needed at TestUnits") + + if args.quiet: + verbose = 0 + else: + verbose = args.verbose + + if not env: + env = os.environ.copy() + + env["VERBOSE"] = f"{verbose}" + + patcher = patch.dict(os.environ, env) + patcher.start() + # ensure it gets stopped after + atexit.register(patcher.stop) + + + if verbose >= 2: + unittest.TextTestRunner(verbosity=verbose).run = lambda suite: suite + + # Load ONLY tests from the calling file + if not suite: + if not pattern: + pattern = caller_file + + loader = unittest.TestLoader() + suite = loader.discover(start_dir=os.path.dirname(caller_file), + pattern=os.path.basename(caller_file)) + + # Flatten the suite for environment injection + tests_to_inject = flatten_suite(suite) + + # Filter tests by method name if -k specified + if args.keyword: + try: + pattern = re.compile(args.keyword) + filtered_suite = unittest.TestSuite() + for test in tests_to_inject: # Use the pre-flattened list + method_name = test.id().split(".")[-1] + if pattern.search(method_name): + filtered_suite.addTest(test) + suite = filtered_suite + except re.error as e: + sys.stderr.write(f"Invalid regex pattern: {e}\n") + sys.exit(1) + else: + # Maintain original suite structure if no keyword filtering + suite = unittest.TestSuite(tests_to_inject) + + if verbose >= 2: + resultclass = None + else: + resultclass = Summary + + runner = unittest.TextTestRunner(verbosity=args.verbose, + resultclass=resultclass, + failfast=args.failfast) + result = runner.run(suite) + if resultclass: + result.printResults(verbose) + + sys.exit(not result.wasSuccessful()) + + +def run_unittest(fname): + """ + Basic usage of TestUnits class. + + Use it when there's no need to pass any extra argument to the tests + with. The recommended way is to place this at the end of each + unittest module:: + + if __name__ == "__main__": + run_unittest(__file__) + """ + TestUnits().run(fname) diff --git a/tools/unittests/kdoc-test-schema.yaml b/tools/unittests/kdoc-test-schema.yaml new file mode 100644 index 000000000000..cf5079711cd8 --- /dev/null +++ b/tools/unittests/kdoc-test-schema.yaml @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2026: Mauro Carvalho Chehab <mchehab@kernel.org>. + +# KDoc Test File Schema + +# This schema contains objects and properties needed to run kernel-doc +# self-tests. + +$schema: "http://json-schema.org/draft-07/schema#" + +tests: + type: array + minItems: 1 + description: | + A list of kernel-doc tests. + + properties: + type: object + properties: + name: + type: string + description: | + Test name. Should be an unique identifier within the schema. + Don't prepend it with "test", as the dynamic test creation will + do it. + + description: + type: string + description: | + Test description + + source: + type: string + description: | + C source code that should be parsed by kernel-doc. + + fname: + type: string + description: | + The filename that contains the element. + When placing real testcases, please use here the name of + the C file (or header) from where the source code was picked. + + exports: + type: array + items: { type: string } + description: | + A list of export identifiers that are expected when parsing source. + + expected: + type: array + minItems: 1 + description: | + A list of expected values. This list consists on objects to check + both kdoc_parser and/or kdoc_output objects. + + items: + type: object + properties: + # + # kdoc_item + # + kdoc_item: + type: object + description: | + Object expected to represent the C source code after parsed + by tools/lib/python/kdoc/kdoc_parser.py KernelDoc class. + See tools/lib/python/kdoc/kdoc_item.py for its contents. + + properties: + name: + type: string + description: | + The name of the identifier (function name, struct name, etc). + type: + type: string + description: | + Type of the object, as filled by kdoc_parser. can be: + - enum + - typedef + - union + - struct + - var + - function + declaration_start_line: + type: integer + description: | + The line number where the kernel-doc markup started. + The first line of the code is line number 1. + sections: + type: object + additionalProperties: { type: string } + description: | + Sections inside the kernel-doc markups: + - "description" + - "return" + - any other part of the markup that starts with "something:" + sections_start_lines: + type: object + additionalProperties: { type: integer } + description: | + a list of section names and the starting line of it. + parameterlist: + type: array + items: { type: string } + description: | + Ordered list of parameter names. + + parameterdesc_start_lines: + type: object + additionalProperties: { type: integer } + description: | + Mapping from parameter name to the line where its + description starts. + parameterdescs: + type: object + additionalProperties: { type: string } + description: | + Mapping from parameter name to its description. + + parametertypes: + type: object + additionalProperties: { type: string } + description: | + Mapping from parameter name to its type. + + other_stuff: + type: object + additionalProperties: {} + description: | + Extra properties that will be stored at the item. + Should match what kdoc_output expects. + + required: + - name + - type + - declaration_start_line + + rst: + type: string + description: | + The expected output for RestOutput class. + + man: + type: string + description: | + The expected output for ManOutput class. + + anyOf: + required: kdoc_item + required: source + + required: + - name + - fname + - expected diff --git a/tools/unittests/kdoc-test.yaml b/tools/unittests/kdoc-test.yaml new file mode 100644 index 000000000000..14d36daa1bba --- /dev/null +++ b/tools/unittests/kdoc-test.yaml @@ -0,0 +1,1698 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) 2026: Mauro Carvalho Chehab <mchehab@kernel.org> + +# Test cases for the dynamic tests. +# Useful to test if kernel-doc classes are doing what it is expected. +# + +tests: +- name: func1 + fname: mock_functions.c + description: "Simplest function test: do nothing, just rst output" + + source: | + /** + * func1 - Not exported function + * @arg1: @arg1 does nothing + * + * Does nothing + * + * return: + * always return 0. + */ + int func1(char *arg1) { return 0; }; + + expected: + - rst: | + .. c:function:: int func1 (char *arg1) + + Not exported function + + .. container:: kernelindent + + **Parameters** + + ``char *arg1`` + **arg1** does nothing + + **Description** + + Does nothing + + **Return** + + always return 0. + + # TODO: how to handle timestamps at .TH? + man: | + .TH "func1" 9 "February 2026" "" "Kernel API Manual" + .SH NAME + func1 \- Not exported function + .SH SYNOPSIS + .B "int" func1 + .BI "(char *arg1 " ");" + .SH ARGUMENTS + .IP "arg1" 12 + \fIarg1\fP does nothing + .SH "DESCRIPTION" + Does nothing + .SH "RETURN" + always return 0. + .SH "SEE ALSO" + .PP + Kernel file \fBmock_functions.c\fR + +- name: func2 + fname: func2.c + description: Simple test with exports + + source: | + /** + * func2() - Exported function + * @arg1: @arg1 does nothing + * + * Does nothing + * + * return: + * always return 0. + */ + int func2(char *arg1) { return 0; }; + EXPORT_SYMBOL(func2); + + exports: func2 + expected: + - kdoc_item: + name: func2 + type: function + declaration_start_line: 1 + + sections: + Description: | + Does nothing + + Return: | + always return 0. + + sections_start_lines: + Description: 3 + Return: 6 + + parameterdescs: + arg1: | + @arg1 does nothing + parameterlist: + - arg1 + parameterdesc_start_lines: + arg1: 2 + parametertypes: + arg1: char *arg1 + + other_stuff: + func_macro: false + functiontype: int + purpose: "Exported function" + typedef: false + + rst: | + .. c:function:: int func2 (char *arg1) + + Exported function + + .. container:: kernelindent + + **Parameters** + + ``char *arg1`` + **arg1** does nothing + + **Description** + + Does nothing + + **Return** + + always return 0. + + man: | + .TH "func2" 9 "February 2026" "" "Kernel API Manual" + .SH NAME + func2 \- Exported function + .SH SYNOPSIS + .B "int" func2 + .BI "(char *arg1 " ");" + .SH ARGUMENTS + .IP "arg1" 12 + \fIarg1\fP does nothing + .SH "DESCRIPTION" + Does nothing + .SH "RETURN" + always return 0. + .SH "SEE ALSO" + .PP + Kernel file \fBfunc2.c\fR + +- name: doc_with_complex_table + description: Test if complex tables are handled + fname: mock.c + source: | + /** + * DOC: Supported input formats and encodings + * + * Depending on the Hardware configuration of the Controller IP, it supports + * a subset of the following input formats and encodings on its internal + * 48bit bus. + * + * +----------------------+----------------------------------+------------------------------+ + * | Format Name | Format Code | Encodings | + * +======================+==================================+==============================+ + * | RGB 4:4:4 8bit | ``MEDIA_BUS_FMT_RGB888_1X24`` | ``V4L2_YCBCR_ENC_DEFAULT`` | + * +----------------------+----------------------------------+------------------------------+ + * | RGB 4:4:4 10bits | ``MEDIA_BUS_FMT_RGB101010_1X30`` | ``V4L2_YCBCR_ENC_DEFAULT`` | + * +----------------------+----------------------------------+------------------------------+ + */ + expected: + - man: | + .TH "Supported input formats and encodings" 9 "March 2026" "" "Kernel API Manual" + .SH "Supported input formats and encodings" + Depending on the Hardware configuration of the Controller IP, it supports + a subset of the following input formats and encodings on its internal + 48bit bus. + .PP + + + .TS + box; + l l l. + \fBFormat Name\fP \fBFormat Code\fP \fBEncodings\fP + _ + RGB 4:4:4 8bit ``MEDIA_BUS_FMT_RGB888_1X24 V4L2_YCBCR_ENC_DEFAULT + RGB 4:4:4 10bits MEDIA_BUS_FMT_RGB101010_1X30 V4L2_YCBCR_ENC_DEFAULT`` + .TE + .SH "SEE ALSO" + .PP + Kernel file \fBmock.c\fR + + rst: |- + .. _Supported input formats and encodings: + **Supported input formats and encodings** + Depending on the Hardware configuration of the Controller IP, it supports + a subset of the following input formats and encodings on its internal + 48bit bus. + +----------------------+----------------------------------+------------------------------+ + | Format Name | Format Code | Encodings | + +======================+==================================+==============================+ + | RGB 4:4:4 8bit | ``MEDIA_BUS_FMT_RGB888_1X24`` | ``V4L2_YCBCR_ENC_DEFAULT`` | + +----------------------+----------------------------------+------------------------------+ + | RGB 4:4:4 10bits | ``MEDIA_BUS_FMT_RGB101010_1X30`` | ``V4L2_YCBCR_ENC_DEFAULT`` | + +----------------------+----------------------------------+------------------------------+ +- name: func_with_ascii_artwork + description: Test if ascii artwork is properly output + fname: mock.c + source: | + /** + * add_cxl_resources() - reflect CXL fixed memory windows in iomem_resource + * @cxl_res: A standalone resource tree where each CXL window is a sibling + * + * Walk each CXL window in @cxl_res and add it to iomem_resource potentially + * expanding its boundaries to ensure that any conflicting resources become + * children. If a window is expanded it may then conflict with a another window + * entry and require the window to be truncated or trimmed. Consider this + * situation:: + * + * |-- "CXL Window 0" --||----- "CXL Window 1" -----| + * |--------------- "System RAM" -------------| + * + * ...where platform firmware has established as System RAM resource across 2 + * windows, but has left some portion of window 1 for dynamic CXL region + * provisioning. In this case "Window 0" will span the entirety of the "System + * RAM" span, and "CXL Window 1" is truncated to the remaining tail past the end + * of that "System RAM" resource. + */ + static int add_cxl_resources(struct resource *cxl_res); + expected: + - man: |- + .TH "add_cxl_resources" 9 "March 2026" "" "Kernel API Manual" + .SH NAME + add_cxl_resources \- reflect CXL fixed memory windows in iomem_resource + .SH SYNOPSIS + .B "int" add_cxl_resources + .BI "(struct resource *cxl_res " ");" + .SH ARGUMENTS + .IP "cxl_res" 12 + A standalone resource tree where each CXL window is a sibling + .SH "DESCRIPTION" + Walk each CXL window in \fIcxl_res\fP and add it to iomem_resource potentially + expanding its boundaries to ensure that any conflicting resources become + children. If a window is expanded it may then conflict with a another window + entry and require the window to be truncated or trimmed. Consider this + situation: + .nf + + |-- "CXL Window 0" --||----- "CXL Window 1" -----| + |--------------- "System RAM" -------------| + + + .fi + .PP + + \&...where platform firmware has established as System RAM resource across 2 + windows, but has left some portion of window 1 for dynamic CXL region + provisioning. In this case "Window 0" will span the entirety of the "System + RAM" span, and "CXL Window 1" is truncated to the remaining tail past the end + of that "System RAM" resource. + .SH "SEE ALSO" + .PP + Kernel file \fBmock.c\fR + rst: | + .. c:function:: int add_cxl_resources (struct resource *cxl_res) + + reflect CXL fixed memory windows in iomem_resource + + .. container:: kernelindent + + **Parameters** + + ``struct resource *cxl_res`` + A standalone resource tree where each CXL window is a sibling + + **Description** + + Walk each CXL window in **cxl_res** and add it to iomem_resource potentially + expanding its boundaries to ensure that any conflicting resources become + children. If a window is expanded it may then conflict with a another window + entry and require the window to be truncated or trimmed. Consider this + situation:: + + |-- "CXL Window 0" --||----- "CXL Window 1" -----| + |--------------- "System RAM" -------------| + + ...where platform firmware has established as System RAM resource across 2 + windows, but has left some portion of window 1 for dynamic CXL region + provisioning. In this case "Window 0" will span the entirety of the "System + RAM" span, and "CXL Window 1" is truncated to the remaining tail past the end + of that "System RAM" resource. + +- name: simple_tables + description: Test formatting two simple tables + fname: mock.c + source: | + /** + * bitmap_onto - translate one bitmap relative to another + * @dst: resulting translated bitmap + * @orig: original untranslated bitmap + * @relmap: bitmap relative to which translated + * @bits: number of bits in each of these bitmaps + * + * =============== ============== ================= + * @orig tmp @dst + * 0 0 40 + * 1 1 41 + * =============== ============== ================= + * + * And: + * + * =============== ============== ================= + * @orig tmp @dst + * =============== ============== ================= + * 9 9 95 + * 10 0 40 [#f1]_ + * =============== ============== ================= + */ + void bitmap_onto(unsigned long *dst, const unsigned long *orig, + const unsigned long *relmap, unsigned int bits); + expected: + - man: | + .TH "bitmap_onto" 9 "March 2026" "" "Kernel API Manual" + .SH NAME + bitmap_onto \- translate one bitmap relative to another + .SH SYNOPSIS + .B "void" bitmap_onto + .BI "(unsigned long *dst " "," + .BI "const unsigned long *orig " "," + .BI "const unsigned long *relmap " "," + .BI "unsigned int bits " ");" + .SH ARGUMENTS + .IP "dst" 12 + resulting translated bitmap + .IP "orig" 12 + original untranslated bitmap + .IP "relmap" 12 + bitmap relative to which translated + .IP "bits" 12 + number of bits in each of these bitmaps + .SH "DESCRIPTION" + + .TS + box; + l l l. + \fIorig\fP tmp \fIdst\fP + 0 0 40 + 1 1 41 + .TE + .PP + + And: + .PP + + + .TS + box; + l l l. + \fIorig\fP tmp \fIdst\fP + .TE + 9 9 95 + 10 0 40 [#f1]_ + .SH "SEE ALSO" + .PP + Kernel file \fBmock.c\fR + + rst: | + .. c:function:: void bitmap_onto (unsigned long *dst, const unsigned long *orig, const unsigned long *relmap, unsigned int bits) + + translate one bitmap relative to another + + .. container:: kernelindent + + **Parameters** + + ``unsigned long *dst`` + resulting translated bitmap + + ``const unsigned long *orig`` + original untranslated bitmap + + ``const unsigned long *relmap`` + bitmap relative to which translated + + ``unsigned int bits`` + number of bits in each of these bitmaps + + **Description** + + =============== ============== ================= + **orig** tmp **dst** + 0 0 40 + 1 1 41 + =============== ============== ================= + + And: + + =============== ============== ================= + **orig** tmp **dst** + =============== ============== ================= + 9 9 95 + 10 0 40 [#f1]_ + =============== ============== ================= + +# +# Variable tests from Randy Dunlap's testset +# +- name: unsigned_long_var_on_uppercase + description: Test an unsigned long varaible in uppercase + fname: mock-vars.c + source: | + /** + * var ROOT_DEV - system root device + * + * @ROOT_DEV is either the successful root device or the root device + * that failed boot in the boot failure message. + */ + unsigned long ROOT_DEV; + expected: + - man: | + .TH "var ROOT_DEV" 9 "February 2026" "" "Kernel API Manual" + .SH NAME + ROOT_DEV \- system root device + .SH SYNOPSIS + unsigned long ROOT_DEV; + .SH "Description" + \fIROOT_DEV\fP is either the successful root device or the root device + that failed boot in the boot failure message. + .SH "SEE ALSO" + .PP + Kernel file \fBmock-vars.c\fR + rst: | + .. c:macro:: ROOT_DEV + + ``unsigned long ROOT_DEV;`` + + system root device + + **Description** + + **ROOT_DEV** is either the successful root device or the root device + that failed boot in the boot failure message. +- name: enum_var + description: Test an enum var with __read_mostly + fname: mock-vars.c + source: | + /** + * var system_state - system state used during boot or suspend/hibernate/resume + * + * @system_state can be used during boot to determine if it is safe to + * make certain calls to other parts of the kernel. It can also be used + * during suspend/hibernate or resume to determine the order of actions + * that need to be executed. The numerical values of system_state are + * sometimes used in numerical ordering tests, so the relative values + * must not be altered. + */ + enum system_states system_state __read_mostly; + expected: + - man: | + .TH "var system_state" 9 "February 2026" "" "Kernel API Manual" + .SH NAME + system_state \- system state used during boot or suspend/hibernate/resume + .SH SYNOPSIS + enum system_states system_state __read_mostly; + .SH "Description" + \fIsystem_state\fP can be used during boot to determine if it is safe to + make certain calls to other parts of the kernel. It can also be used + during suspend/hibernate or resume to determine the order of actions + that need to be executed. The numerical values of system_state are + sometimes used in numerical ordering tests, so the relative values + must not be altered. + .SH "SEE ALSO" + .PP + Kernel file \fBmock-vars.c\fR + rst: | + .. c:macro:: system_state + + ``enum system_states system_state __read_mostly;`` + + system state used during boot or suspend/hibernate/resume + + **Description** + + **system_state** can be used during boot to determine if it is safe to + make certain calls to other parts of the kernel. It can also be used + during suspend/hibernate or resume to determine the order of actions + that need to be executed. The numerical values of system_state are + sometimes used in numerical ordering tests, so the relative values + must not be altered. +- name: char_pointer_var + description: Test char * var with __ro_after_init + fname: mock-vars.c + source: | + /** + * var saved_command_line - kernel's command line, saved from use at + * any later time in the kernel. + */ + char *saved_command_line __ro_after_init; + expected: + - man: | + .TH "var saved_command_line" 9 "February 2026" "" "Kernel API Manual" + .SH NAME + saved_command_line \- kernel's command line, saved from use at any later time in the kernel. + .SH SYNOPSIS + char *saved_command_line __ro_after_init; + .SH "SEE ALSO" + .PP + Kernel file \fBmock-vars.c\fR + rst: | + .. c:macro:: saved_command_line + + ``char *saved_command_line __ro_after_init;`` + + kernel's command line, saved from use at any later time in the kernel. +- name: unsigned_long_with_default + description: Test an unsigned long var that is set to a default value + fname: mock-vars.c + source: | + /** + * var loop_per_jiffy - calculated loop count needed to consume one jiffy + * of time + */ + unsigned long loops_per_jiffy = (1<<12); + expected: + - man: | + .TH "var loops_per_jiffy" 9 "February 2026" "" "Kernel API Manual" + .SH NAME + loops_per_jiffy \- calculated loop count needed to consume one jiffy of time + .SH SYNOPSIS + unsigned long loops_per_jiffy = (1<<12); + .SH "Initialization" + default: (1<<12) + .SH "SEE ALSO" + .PP + Kernel file \fBmock-vars.c\fR + rst: | + .. c:macro:: loops_per_jiffy + + ``unsigned long loops_per_jiffy = (1<<12);`` + + calculated loop count needed to consume one jiffy of time + + **Initialization** + + default: ``(1<<12)`` +- name: unsigned_long + description: test a simple unsigned long variable. + fname: mock-vars.c + source: | + /** + * var preset_lpj - lpj (loops per jiffy) value set from kernel + * command line using "lpj=VALUE" + * + * See Documentation/admin-guide/kernel-parameters.txt ("lpj=") for details. + */ + unsigned long preset_lpj; + expected: + - man: | + .TH "var preset_lpj" 9 "February 2026" "" "Kernel API Manual" + .SH NAME + preset_lpj \- lpj (loops per jiffy) value set from kernel command line using "lpj=VALUE" + .SH SYNOPSIS + unsigned long preset_lpj; + .SH "Description" + See Documentation/admin-guide/kernel-parameters.txt ("lpj=") for details. + .SH "SEE ALSO" + .PP + Kernel file \fBmock-vars.c\fR + rst: | + .. c:macro:: preset_lpj + + ``unsigned long preset_lpj;`` + + lpj (loops per jiffy) value set from kernel command line using "lpj=VALUE" + + **Description** + + See Documentation/admin-guide/kernel-parameters.txt ("lpj=") for details. +- name: char_array + description: test a char array variable + fname: mock-vars.c + source: | + /** + * var linux_proc_banner - text used from /proc/version file + * + * * first %s is sysname (e.g., "Linux") + * * second %s is release + * * third %s is version + */ + char linux_proc_banner[]; + expected: + - man: | + .TH "var linux_proc_banner" 9 "February 2026" "" "Kernel API Manual" + .SH NAME + linux_proc_banner \- text used from /proc/version file + .SH SYNOPSIS + char linux_proc_banner[]; + .SH "Description" + .IP \[bu] + first s is sysname (e.g., "Linux") + .IP \[bu] + second s is release + .IP \[bu] + third s is version + .SH "SEE ALSO" + .PP + Kernel file \fBmock-vars.c\fR + rst: | + .. c:macro:: linux_proc_banner + + ``char linux_proc_banner[];`` + + text used from /proc/version file + + **Description** + + * first ``s`` is sysname (e.g., "Linux") + * second ``s`` is release + * third ``s`` is version +- name: const_char_array + description: test a const char array variable + fname: mock-vars.c + source: | + /** + * var linux_banner - Linux boot banner, usually printed at boot time + */ + const char linux_banner[]; + expected: + - man: | + .TH "var linux_banner" 9 "February 2026" "" "Kernel API Manual" + .SH NAME + linux_banner \- Linux boot banner, usually printed at boot time + .SH SYNOPSIS + const char linux_banner[]; + .SH "SEE ALSO" + .PP + Kernel file \fBmock-vars.c\fR + rst: | + .. c:macro:: linux_banner + + ``const char linux_banner[];`` + + Linux boot banner, usually printed at boot time +- name: static_atomic64_t_var + description: test a static atomi64_t variable + fname: mock-vars.c + source: | + /** + * var diskseq - unique sequence number for block device instances + * + * Allows userspace to associate uevents to the lifetime of a device + */ + static atomic64_t diskseq; + expected: + - man: | + .TH "var diskseq" 9 "February 2026" "" "Kernel API Manual" + .SH NAME + diskseq \- unique sequence number for block device instances + .SH SYNOPSIS + static atomic64_t diskseq; + .SH "Description" + Allows userspace to associate uevents to the lifetime of a device + .SH "SEE ALSO" + .PP + Kernel file \fBmock-vars.c\fR + rst: | + .. c:macro:: diskseq + + ``static atomic64_t diskseq;`` + + unique sequence number for block device instances + + **Description** + + Allows userspace to associate uevents to the lifetime of a device +- name: unsigned_long_on_init + description: test an unsigned long var at "init" with a different timestamp. + fname: init/mock-vars.c + source: | + /** + * var rtnl_mutex - historical global lock for networking control operations. + * + * @rtnl_mutex is used to serialize rtnetlink requests + * and protect all kernel internal data structures related to networking. + * + * See Documentation/networking/netdevices.rst for details. + * Often known as the rtnl_lock, although rtnl_lock is a kernel function. + */ + unsigned long rtnl_mutex; + expected: + - man: | + .TH "var rtnl_mutex" 9 "February 2026" "init" "Kernel API Manual" + .SH NAME + rtnl_mutex \- historical global lock for networking control operations. + .SH SYNOPSIS + unsigned long rtnl_mutex; + .SH "Description" + \fIrtnl_mutex\fP is used to serialize rtnetlink requests + and protect all kernel internal data structures related to networking. + .PP + + See Documentation/networking/netdevices.rst for details. + Often known as the rtnl_lock, although rtnl_lock is a kernel function. + .SH "SEE ALSO" + .PP + Kernel file \fBinit/mock-vars.c\fR + rst: | + .. c:macro:: rtnl_mutex + + ``unsigned long rtnl_mutex;`` + + historical global lock for networking control operations. + + **Description** + + **rtnl_mutex** is used to serialize rtnetlink requests + and protect all kernel internal data structures related to networking. + + See Documentation/networking/netdevices.rst for details. + Often known as the rtnl_lock, although rtnl_lock is a kernel function. + + +- name: struct_kcov + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * struct kcov - kcov descriptor (one per opened debugfs file). + * State transitions of the descriptor: + * + * - initial state after open() + * - then there must be a single ioctl(KCOV_INIT_TRACE) call + * - then, mmap() call (several calls are allowed but not useful) + * - then, ioctl(KCOV_ENABLE, arg), where arg is + * KCOV_TRACE_PC - to trace only the PCs + * or + * KCOV_TRACE_CMP - to trace only the comparison operands + * - then, ioctl(KCOV_DISABLE) to disable the task. + * + * Enabling/disabling ioctls can be repeated (only one task a time allowed). + */ + struct kcov { + /** + * @refcount: Reference counter. We keep one for: + * - opened file descriptor + * - task with enabled coverage (we can't unwire it from another task) + * - each code section for remote coverage collection + */ + refcount_t refcount; + /** + * @lock: The lock protects mode, size, area and t. + */ + spinlock_t lock; + /** + * @mode: the kcov_mode + */ + enum kcov_mode mode __guarded_by(&lock); + /** + * @size: Size of arena (in long's). + */ + unsigned int size __guarded_by(&lock); + /** + * @area: Coverage buffer shared with user space. + */ + void *area __guarded_by(&lock); + /** + * @t: Task for which we collect coverage, or NULL. + */ + struct task_struct *t __guarded_by(&lock); + /** + * @remote: Collecting coverage from remote (background) threads. + */ + bool remote; + /** + * @remote_size: Size of remote area (in long's). + */ + unsigned int remote_size; + /** + * @sequence: Sequence is incremented each time kcov is reenabled, + * used by kcov_remote_stop(), see the comment there. + */ + int sequence; + }; + expected: + - man: | + .TH "struct kcov" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + struct kcov \- kcov descriptor (one per opened debugfs file). State transitions of the descriptor: + .SH SYNOPSIS + struct kcov { + .br + .BI " refcount_t refcount;" + .br + .BI " spinlock_t lock;" + .br + .BI " enum kcov_mode mode;" + .br + .BI " unsigned int size;" + .br + .BI " void *area;" + .br + .BI " struct task_struct *t;" + .br + .BI " bool remote;" + .br + .BI " unsigned int remote_size;" + .br + .BI " int sequence;" + .br + .BI " + }; + .br + + .SH Members + .IP "refcount" 12 + Reference counter. We keep one for: + .IP \[bu] + opened file descriptor + .IP \[bu] + task with enabled coverage (we can't unwire it from another task) + .IP \[bu] + each code section for remote coverage collection + .IP "lock" 12 + The lock protects mode, size, area and t. + .IP "mode" 12 + the kcov_mode + .IP "size" 12 + Size of arena (in long's). + .IP "area" 12 + Coverage buffer shared with user space. + .IP "t" 12 + Task for which we collect coverage, or NULL. + .IP "remote" 12 + Collecting coverage from remote (background) threads. + .IP "remote_size" 12 + Size of remote area (in long's). + .IP "sequence" 12 + Sequence is incremented each time kcov is reenabled, + used by \fBkcov_remote_stop\fP, see the comment there. + .SH "Description" + .IP \[bu] + initial state after \fBopen\fP + .IP \[bu] + then there must be a single ioctl(KCOV_INIT_TRACE) call + .IP \[bu] + then, \fBmmap\fP call (several calls are allowed but not useful) + .IP \[bu] + then, ioctl(KCOV_ENABLE, arg), where arg is + KCOV_TRACE_PC - to trace only the PCs + or + KCOV_TRACE_CMP - to trace only the comparison operands + .IP \[bu] + then, ioctl(KCOV_DISABLE) to disable the task. + .PP + + Enabling/disabling ioctls can be repeated (only one task a time allowed). + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:struct:: kcov + + kcov descriptor (one per opened debugfs file). State transitions of the descriptor: + + .. container:: kernelindent + + **Definition**:: + + struct kcov { + refcount_t refcount; + spinlock_t lock; + enum kcov_mode mode; + unsigned int size; + void *area; + struct task_struct *t; + bool remote; + unsigned int remote_size; + int sequence; + }; + + **Members** + + ``refcount`` + Reference counter. We keep one for: + - opened file descriptor + - task with enabled coverage (we can't unwire it from another task) + - each code section for remote coverage collection + + ``lock`` + The lock protects mode, size, area and t. + + ``mode`` + the kcov_mode + + ``size`` + Size of arena (in long's). + + ``area`` + Coverage buffer shared with user space. + + ``t`` + Task for which we collect coverage, or NULL. + + ``remote`` + Collecting coverage from remote (background) threads. + + ``remote_size`` + Size of remote area (in long's). + + ``sequence`` + Sequence is incremented each time kcov is reenabled, + used by kcov_remote_stop(), see the comment there. + + + **Description** + + - initial state after open() + - then there must be a single ioctl(KCOV_INIT_TRACE) call + - then, mmap() call (several calls are allowed but not useful) + - then, ioctl(KCOV_ENABLE, arg), where arg is + KCOV_TRACE_PC - to trace only the PCs + or + KCOV_TRACE_CMP - to trace only the comparison operands + - then, ioctl(KCOV_DISABLE) to disable the task. + + Enabling/disabling ioctls can be repeated (only one task a time allowed). + +- name: pool_offset + description: mock_tests/kdoc-drop-ctx-lock.c line 83 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * var pool_offset - Offset to the unused space in the currently used pool. + * + */ + size_t pool_offset __guarded_by(&pool_lock) = DEPOT_POOL_SIZE; + expected: + - man: | + .TH "var pool_offset" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + pool_offset \- Offset to the unused space in the currently used pool. + .SH SYNOPSIS + size_t pool_offset __guarded_by(&pool_lock) = DEPOT_POOL_SIZE; + .SH "Initialization" + default: DEPOT_POOL_SIZE + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:macro:: pool_offset + + ``size_t pool_offset __guarded_by(&pool_lock) = DEPOT_POOL_SIZE;`` + + Offset to the unused space in the currently used pool. + + **Initialization** + + default: ``DEPOT_POOL_SIZE`` +- name: free_stacks + description: mock_tests/kdoc-drop-ctx-lock.c line 88 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * var free_stacks - Freelist of stack records within stack_pools. + * + */ + __guarded_by(&pool_lock) LIST_HEAD(free_stacks); + expected: + - man: | + .TH "var free_stacks" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + free_stacks \- Freelist of stack records within stack_pools. + .SH SYNOPSIS + __guarded_by(&pool_lock) LIST_HEAD(free_stacks); + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:macro:: free_stacks + + ``__guarded_by(&pool_lock) LIST_HEAD(free_stacks);`` + + Freelist of stack records within stack_pools. +- name: stack_pools + description: mock_tests/kdoc-drop-ctx-lock.c line 94 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * var stack_pools - Array of memory regions that store stack records. + * + */ + void **stack_pools __pt_guarded_by(&pool_lock); + expected: + - man: | + .TH "var stack_pools" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + stack_pools \- Array of memory regions that store stack records. + .SH SYNOPSIS + void **stack_pools __pt_guarded_by(&pool_lock); + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:macro:: stack_pools + + ``void **stack_pools __pt_guarded_by(&pool_lock);`` + + Array of memory regions that store stack records. +- name: prepare_report_consumer + description: mock_tests/kdoc-drop-ctx-lock.c line 103 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * prepare_report_consumer - prepare the report consumer + * @flags: flags + * @ai: not that AI + * @other_info: yes that + */ + bool prepare_report_consumer(unsigned long *flags, + const struct access_info *ai, + struct other_info *other_info) + __cond_acquires(true, &report_lock) + { + expected: + - man: | + .TH "prepare_report_consumer" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + prepare_report_consumer \- prepare the report consumer + .SH SYNOPSIS + .B "bool" prepare_report_consumer + .BI "(unsigned long *flags " "," + .BI "const struct access_info *ai " "," + .BI "struct other_info *other_info " ");" + .SH ARGUMENTS + .IP "flags" 12 + flags + .IP "ai" 12 + not that AI + .IP "other_info" 12 + yes that + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: bool prepare_report_consumer (unsigned long *flags, const struct access_info *ai, struct other_info *other_info) + + prepare the report consumer + + .. container:: kernelindent + + **Parameters** + + ``unsigned long *flags`` + flags + + ``const struct access_info *ai`` + not that AI + + ``struct other_info *other_info`` + yes that +- name: tcp_sigpool_start + description: mock_tests/kdoc-drop-ctx-lock.c line 117 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * tcp_sigpool_start - start a tcp message of @id, using @c + * @id: TCP message ID + * @c: the &tcp_sigpool to use + */ + int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c) __cond_acquires(0, RCU_BH) + { + expected: + - man: | + .TH "tcp_sigpool_start" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + tcp_sigpool_start \- start a tcp message of @id, using @c + .SH SYNOPSIS + .B "int" tcp_sigpool_start + .BI "(unsigned int id " "," + .BI "struct tcp_sigpool *c " ");" + .SH ARGUMENTS + .IP "id" 12 + TCP message ID + .IP "c" 12 + the \fItcp_sigpool\fP to use + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: int tcp_sigpool_start (unsigned int id, struct tcp_sigpool *c) + + start a tcp message of **id**, using **c** + + .. container:: kernelindent + + **Parameters** + + ``unsigned int id`` + TCP message ID + + ``struct tcp_sigpool *c`` + the :c:type:`tcp_sigpool` to use +- name: undo_report_consumer + description: mock_tests/kdoc-drop-ctx-lock.c line 129 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * undo_report_consumer - teardown a report consumer + * @flags: those flags + * @ai: not that AI + * @other_info: yes that + */ + bool undo_report_consumer(unsigned long *flags, + const struct access_info *ai, + struct other_info *other_info) + __cond_releases(true, &report_lock) + { + expected: + - man: | + .TH "undo_report_consumer" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + undo_report_consumer \- teardown a report consumer + .SH SYNOPSIS + .B "bool" undo_report_consumer + .BI "(unsigned long *flags " "," + .BI "const struct access_info *ai " "," + .BI "struct other_info *other_info " ");" + .SH ARGUMENTS + .IP "flags" 12 + those flags + .IP "ai" 12 + not that AI + .IP "other_info" 12 + yes that + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: bool undo_report_consumer (unsigned long *flags, const struct access_info *ai, struct other_info *other_info) + + teardown a report consumer + + .. container:: kernelindent + + **Parameters** + + ``unsigned long *flags`` + those flags + + ``const struct access_info *ai`` + not that AI + + ``struct other_info *other_info`` + yes that +- name: debugfs_enter_cancellation + description: mock_tests/kdoc-drop-ctx-lock.c line 143 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * debugfs_enter_cancellation - begin a cancellation operation on @file + * @file: the target file + * @cancellation: the operation to execute + */ + void debugfs_enter_cancellation(struct file *file, + struct debugfs_cancellation *cancellation) __acquires(cancellation) + { } + expected: + - man: | + .TH "debugfs_enter_cancellation" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + debugfs_enter_cancellation \- begin a cancellation operation on @file + .SH SYNOPSIS + .B "void" debugfs_enter_cancellation + .BI "(struct file *file " "," + .BI "struct debugfs_cancellation *cancellation " ");" + .SH ARGUMENTS + .IP "file" 12 + the target file + .IP "cancellation" 12 + the operation to execute + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: void debugfs_enter_cancellation (struct file *file, struct debugfs_cancellation *cancellation) + + begin a cancellation operation on **file** + + .. container:: kernelindent + + **Parameters** + + ``struct file *file`` + the target file + + ``struct debugfs_cancellation *cancellation`` + the operation to execute +- name: debugfs_leave_cancellation + description: mock_tests/kdoc-drop-ctx-lock.c line 152 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * debugfs_leave_cancellation - wrapup the cancellation operation on @file + * @file: the target file + * @cancellation: the operation to wrapup + */ + void debugfs_leave_cancellation(struct file *file, + struct debugfs_cancellation *cancellation) __releases(cancellation) + { } + expected: + - man: | + .TH "debugfs_leave_cancellation" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + debugfs_leave_cancellation \- wrapup the cancellation operation on @file + .SH SYNOPSIS + .B "void" debugfs_leave_cancellation + .BI "(struct file *file " "," + .BI "struct debugfs_cancellation *cancellation " ");" + .SH ARGUMENTS + .IP "file" 12 + the target file + .IP "cancellation" 12 + the operation to wrapup + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: void debugfs_leave_cancellation (struct file *file, struct debugfs_cancellation *cancellation) + + wrapup the cancellation operation on **file** + + .. container:: kernelindent + + **Parameters** + + ``struct file *file`` + the target file + + ``struct debugfs_cancellation *cancellation`` + the operation to wrapup +- name: acpi_os_acquire_lock + description: mock_tests/kdoc-drop-ctx-lock.c line 161 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * acpi_os_acquire_lock - Acquire a spinlock. + * @lockp: pointer to the spinlock_t. + */ + acpi_cpu_flags acpi_os_acquire_lock(acpi_spinlock lockp) + __acquires(lockp) + { + expected: + - man: | + .TH "acpi_os_acquire_lock" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + acpi_os_acquire_lock \- Acquire a spinlock. + .SH SYNOPSIS + .B "acpi_cpu_flags" acpi_os_acquire_lock + .BI "(acpi_spinlock lockp " ");" + .SH ARGUMENTS + .IP "lockp" 12 + pointer to the spinlock_t. + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: acpi_cpu_flags acpi_os_acquire_lock (acpi_spinlock lockp) + + Acquire a spinlock. + + .. container:: kernelindent + + **Parameters** + + ``acpi_spinlock lockp`` + pointer to the spinlock_t. +- name: acpi_os_release_lock + description: mock_tests/kdoc-drop-ctx-lock.c line 172 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * acpi_os_release_lock - Release a spinlock. + * @lockp: pointer to the spinlock_t. + * @not_used: these flags are not used. + */ + void acpi_os_release_lock(acpi_spinlock lockp, acpi_cpu_flags not_used) + __releases(lockp) + { + expected: + - man: | + .TH "acpi_os_release_lock" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + acpi_os_release_lock \- Release a spinlock. + .SH SYNOPSIS + .B "void" acpi_os_release_lock + .BI "(acpi_spinlock lockp " "," + .BI "acpi_cpu_flags not_used " ");" + .SH ARGUMENTS + .IP "lockp" 12 + pointer to the spinlock_t. + .IP "not_used" 12 + these flags are not used. + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: void acpi_os_release_lock (acpi_spinlock lockp, acpi_cpu_flags not_used) + + Release a spinlock. + + .. container:: kernelindent + + **Parameters** + + ``acpi_spinlock lockp`` + pointer to the spinlock_t. + + ``acpi_cpu_flags not_used`` + these flags are not used. +- name: tx + description: mock_tests/kdoc-drop-ctx-lock.c line 183 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * tx - transmit message ID @id + * @id: message ID to transmit + */ + int tx(int id) __must_hold(&txlock) + { + expected: + - man: | + .TH "tx" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + tx \- transmit message ID @id + .SH SYNOPSIS + .B "int" tx + .BI "(int id " ");" + .SH ARGUMENTS + .IP "id" 12 + message ID to transmit + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: int tx (int id) + + transmit message ID **id** + + .. container:: kernelindent + + **Parameters** + + ``int id`` + message ID to transmit +- name: contend_for_bm + description: mock_tests/kdoc-drop-ctx-lock.c line 192 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * contend_for_bm - try to become the bus master + * @card: the &fw_card (describes the bus) + */ + enum bm_contention_outcome contend_for_bm(struct fw_card *card) + __must_hold(&card->lock) + { + expected: + - man: | + .TH "contend_for_bm" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + contend_for_bm \- try to become the bus master + .SH SYNOPSIS + .B "enum bm_contention_outcome" contend_for_bm + .BI "(struct fw_card *card " ");" + .SH ARGUMENTS + .IP "card" 12 + the \fIfw_card\fP (describes the bus) + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: enum bm_contention_outcome contend_for_bm (struct fw_card *card) + + try to become the bus master + + .. container:: kernelindent + + **Parameters** + + ``struct fw_card *card`` + the :c:type:`fw_card` (describes the bus) +- name: prepare_report_producer + description: mock_tests/kdoc-drop-ctx-lock.c line 202 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * prepare_report_producer - prepare the report producer + * @flags: still flags + * @ai: some AI + * @other_info: Populate @other_info; requires that the provided + * @other_info not in use. + */ + void prepare_report_producer(unsigned long *flags, + const struct access_info *ai, + struct other_info *other_info) + __must_not_hold(&report_lock) + { } + expected: + - man: | + .TH "prepare_report_producer" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + prepare_report_producer \- prepare the report producer + .SH SYNOPSIS + .B "void" prepare_report_producer + .BI "(unsigned long *flags " "," + .BI "const struct access_info *ai " "," + .BI "struct other_info *other_info " ");" + .SH ARGUMENTS + .IP "flags" 12 + still flags + .IP "ai" 12 + some AI + .IP "other_info" 12 + Populate \fIother_info\fP; requires that the provided + \fIother_info\fP not in use. + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: void prepare_report_producer (unsigned long *flags, const struct access_info *ai, struct other_info *other_info) + + prepare the report producer + + .. container:: kernelindent + + **Parameters** + + ``unsigned long *flags`` + still flags + + ``const struct access_info *ai`` + some AI + + ``struct other_info *other_info`` + Populate **other_info**; requires that the provided + **other_info** not in use. +- name: crypto_alg_lookup + description: mock_tests/kdoc-drop-ctx-lock.c line 215 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * __crypto_alg_lookup() - lookup the algorithm by name/type/mask + * @name: name to search for + * @type: type to search for + * @mask: mask to match + */ + struct crypto_alg *__crypto_alg_lookup(const char *name, u32 type, + u32 mask) + __must_hold_shared(&crypto_alg_sem) + { + expected: + - man: | + .TH "__crypto_alg_lookup" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + __crypto_alg_lookup \- lookup the algorithm by name/type/mask + .SH SYNOPSIS + .B "struct crypto_alg *" __crypto_alg_lookup + .BI "(const char *name " "," + .BI "u32 type " "," + .BI "u32 mask " ");" + .SH ARGUMENTS + .IP "name" 12 + name to search for + .IP "type" 12 + type to search for + .IP "mask" 12 + mask to match + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: struct crypto_alg * __crypto_alg_lookup (const char *name, u32 type, u32 mask) + + lookup the algorithm by name/type/mask + + .. container:: kernelindent + + **Parameters** + + ``const char *name`` + name to search for + + ``u32 type`` + type to search for + + ``u32 mask`` + mask to match +- name: down_read_trylock + description: mock_tests/kdoc-drop-ctx-lock.c line 228 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * down_read_trylock - trylock for reading + * @sem: the semaphore to try to lock + * + * Returns: 1 if successful, 0 if contention + */ + extern int down_read_trylock(struct rw_semaphore *sem) __cond_acquires_shared(true, sem); + expected: + - man: | + .TH "down_read_trylock" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + down_read_trylock \- trylock for reading + .SH SYNOPSIS + .B "int" down_read_trylock + .BI "(struct rw_semaphore *sem " ");" + .SH ARGUMENTS + .IP "sem" 12 + the semaphore to try to lock + .SH "RETURN" + 1 if successful, 0 if contention + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: int down_read_trylock (struct rw_semaphore *sem) + + trylock for reading + + .. container:: kernelindent + + **Parameters** + + ``struct rw_semaphore *sem`` + the semaphore to try to lock + + **Return** + + 1 if successful, 0 if contention +- name: tomoyo_read_lock + description: mock_tests/kdoc-drop-ctx-lock.c line 236 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * tomoyo_read_lock - Take lock for protecting policy. + * + * Returns: index number for tomoyo_read_unlock(). + */ + int tomoyo_read_lock(void) + __acquires_shared(&tomoyo_ss) + { + expected: + - man: | + .TH "tomoyo_read_lock" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + tomoyo_read_lock \- Take lock for protecting policy. + .SH SYNOPSIS + .B "int" tomoyo_read_lock + .BI "(void " ");" + .SH ARGUMENTS + .IP "void" 12 + no arguments + .SH "RETURN" + index number for \fBtomoyo_read_unlock\fP. + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: int tomoyo_read_lock (void) + + Take lock for protecting policy. + + .. container:: kernelindent + + **Parameters** + + ``void`` + no arguments + + **Return** + + index number for tomoyo_read_unlock(). +- name: tomoyo_read_unlock + description: mock_tests/kdoc-drop-ctx-lock.c line 247 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * tomoyo_read_unlock - Release lock for protecting policy. + * + * @idx: Index number returned by tomoyo_read_lock(). + */ + void tomoyo_read_unlock(int idx) + __releases_shared(&tomoyo_ss) + { } + expected: + - man: | + .TH "tomoyo_read_unlock" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + tomoyo_read_unlock \- Release lock for protecting policy. + .SH SYNOPSIS + .B "void" tomoyo_read_unlock + .BI "(int idx " ");" + .SH ARGUMENTS + .IP "idx" 12 + Index number returned by \fBtomoyo_read_lock\fP. + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: void tomoyo_read_unlock (int idx) + + Release lock for protecting policy. + + .. container:: kernelindent + + **Parameters** + + ``int idx`` + Index number returned by tomoyo_read_lock(). +- name: c_stop + description: mock_tests/kdoc-drop-ctx-lock.c line 256 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * c_stop - stop the seq_file iteration + * @m: the &struct seq_file + * @p: handle + */ + void c_stop(struct seq_file *m, void *p) + __releases_shared(&crypto_alg_sem) + { } + expected: + - man: | + .TH "c_stop" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + c_stop \- stop the seq_file iteration + .SH SYNOPSIS + .B "void" c_stop + .BI "(struct seq_file *m " "," + .BI "void *p " ");" + .SH ARGUMENTS + .IP "m" 12 + the \fIstruct seq_file\fP + .IP "p" 12 + handle + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: void c_stop (struct seq_file *m, void *p) + + stop the seq_file iteration + + .. container:: kernelindent + + **Parameters** + + ``struct seq_file *m`` + the :c:type:`struct seq_file <seq_file>` + + ``void *p`` + handle +- name: spin_lock + description: mock_tests/kdoc-drop-ctx-lock.c line 265 + fname: mock_tests/kdoc-drop-ctx-lock.c + source: | + /** + * spin_lock - spin until the @lock is acquired + * @lock: the spinlock + */ + void spin_lock(spinlock_t *lock) + __acquires(lock) __no_context_analysis + { } + expected: + - man: | + .TH "spin_lock" 9 "February 2026" "mock_tests" "Kernel API Manual" + .SH NAME + spin_lock \- spin until the @lock is acquired + .SH SYNOPSIS + .B "void" spin_lock + .BI "(spinlock_t *lock " ");" + .SH ARGUMENTS + .IP "lock" 12 + the spinlock + .SH "SEE ALSO" + .PP + Kernel file \fBmock_tests/kdoc-drop-ctx-lock.c\fR + rst: | + .. c:function:: void spin_lock (spinlock_t *lock) + + spin until the **lock** is acquired + + .. container:: kernelindent + + **Parameters** + + ``spinlock_t *lock`` + the spinlock diff --git a/tools/unittests/run.py b/tools/unittests/run.py new file mode 100755 index 000000000000..8c19036d43a1 --- /dev/null +++ b/tools/unittests/run.py @@ -0,0 +1,17 @@ +#!/bin/env python3 +import os +import unittest +import sys + +TOOLS_DIR=os.path.join(os.path.dirname(os.path.realpath(__file__)), "..") +sys.path.insert(0, TOOLS_DIR) + +from lib.python.unittest_helper import TestUnits + +if __name__ == "__main__": + loader = unittest.TestLoader() + + suite = loader.discover(start_dir=os.path.join(TOOLS_DIR, "unittests"), + pattern="test*.py") + + TestUnits().run("", suite=suite) diff --git a/tools/unittests/test_cmatch.py b/tools/unittests/test_cmatch.py new file mode 100755 index 000000000000..7b996f83784d --- /dev/null +++ b/tools/unittests/test_cmatch.py @@ -0,0 +1,821 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2026: Mauro Carvalho Chehab <mchehab@kernel.org>. +# +# pylint: disable=C0413,R0904 + + +""" +Unit tests for kernel-doc CMatch. +""" + +import os +import re +import sys +import unittest + + +# Import Python modules + +SRC_DIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(SRC_DIR, "../lib/python")) + +from kdoc.c_lex import CMatch +from kdoc.kdoc_re import KernRe +from unittest_helper import run_unittest + +# +# Override unittest.TestCase to better compare diffs ignoring whitespaces +# +class TestCaseDiff(unittest.TestCase): + """ + Disable maximum limit on diffs and add a method to better + handle diffs with whitespace differences. + """ + + @classmethod + def setUpClass(cls): + """Ensure that there won't be limit for diffs""" + cls.maxDiff = None + + +# +# Tests doing with different macros +# + +class TestSearch(TestCaseDiff): + """ + Test search mechanism + """ + + def test_search_acquires_simple(self): + line = "__acquires(ctx) foo();" + result = ", ".join(CMatch("__acquires").search(line)) + self.assertEqual(result, "__acquires(ctx)") + + def test_search_acquires_multiple(self): + line = "__acquires(ctx) __acquires(other) bar();" + result = ", ".join(CMatch("__acquires").search(line)) + self.assertEqual(result, "__acquires(ctx), __acquires(other)") + + def test_search_acquires_nested_paren(self): + line = "__acquires((ctx1, ctx2)) baz();" + result = ", ".join(CMatch("__acquires").search(line)) + self.assertEqual(result, "__acquires((ctx1, ctx2))") + + def test_search_must_hold(self): + line = "__must_hold(&lock) do_something();" + result = ", ".join(CMatch("__must_hold").search(line)) + self.assertEqual(result, "__must_hold(&lock)") + + def test_search_must_hold_shared(self): + line = "__must_hold_shared(RCU) other();" + result = ", ".join(CMatch("__must_hold_shared").search(line)) + self.assertEqual(result, "__must_hold_shared(RCU)") + + def test_search_no_false_positive(self): + line = "call__acquires(foo); // should stay intact" + result = ", ".join(CMatch(r"__acquires").search(line)) + self.assertEqual(result, "") + + def test_search_no_macro_remains(self): + line = "do_something_else();" + result = ", ".join(CMatch("__acquires").search(line)) + self.assertEqual(result, "") + + def test_search_no_function(self): + line = "something" + result = ", ".join(CMatch(line).search(line)) + self.assertEqual(result, "") + +# +# Override unittest.TestCase to better compare diffs ignoring whitespaces +# +class TestCaseDiff(unittest.TestCase): + """ + Disable maximum limit on diffs and add a method to better + handle diffs with whitespace differences. + """ + + @classmethod + def setUpClass(cls): + """Ensure that there won't be limit for diffs""" + cls.maxDiff = None + + def assertLogicallyEqual(self, a, b): + """ + Compare two results ignoring multiple whitespace differences. + + This is useful to check more complex matches picked from examples. + On a plus side, we also don't need to use dedent. + Please notice that line breaks still need to match. We might + remove it at the regex, but this way, checking the diff is easier. + """ + a = re.sub(r"[\t ]+", " ", a.strip()) + b = re.sub(r"[\t ]+", " ", b.strip()) + + a = re.sub(r"\s+\n", "\n", a) + b = re.sub(r"\s+\n", "\n", b) + + a = re.sub(" ;", ";", a) + b = re.sub(" ;", ";", b) + + self.assertEqual(a, b) + +# +# Tests doing with different macros +# + +class TestSubMultipleMacros(TestCaseDiff): + """ + Tests doing with different macros. + + Here, we won't use assertLogicallyEqual. Instead, we'll check if each + of the expected patterns are present at the answer. + """ + + def test_acquires_simple(self): + """Simple replacement test with __acquires""" + line = "__acquires(ctx) foo();" + result = CMatch(r"__acquires").sub("REPLACED", line) + + self.assertEqual("REPLACED foo();", result) + + def test_acquires_multiple(self): + """Multiple __acquires""" + line = "__acquires(ctx) __acquires(other) bar();" + result = CMatch(r"__acquires").sub("REPLACED", line) + + self.assertEqual("REPLACED REPLACED bar();", result) + + def test_acquires_nested_paren(self): + """__acquires with nested pattern""" + line = "__acquires((ctx1, ctx2)) baz();" + result = CMatch(r"__acquires").sub("REPLACED", line) + + self.assertEqual("REPLACED baz();", result) + + def test_must_hold(self): + """__must_hold with a pointer""" + line = "__must_hold(&lock) do_something();" + result = CMatch(r"__must_hold").sub("REPLACED", line) + + self.assertNotIn("__must_hold(", result) + self.assertIn("do_something();", result) + + def test_must_hold_shared(self): + """__must_hold with an upercase defined value""" + line = "__must_hold_shared(RCU) other();" + result = CMatch(r"__must_hold_shared").sub("REPLACED", line) + + self.assertNotIn("__must_hold_shared(", result) + self.assertIn("other();", result) + + def test_no_false_positive(self): + """ + Ensure that unrelated text containing similar patterns is preserved + """ + line = "call__acquires(foo); // should stay intact" + result = CMatch(r"\b__acquires").sub("REPLACED", line) + + self.assertLogicallyEqual(result, "call__acquires(foo);") + + def test_mixed_macros(self): + """Add a mix of macros""" + line = "__acquires(ctx) __releases(ctx) __must_hold(&lock) foo();" + + result = CMatch(r"__acquires").sub("REPLACED", line) + result = CMatch(r"__releases").sub("REPLACED", result) + result = CMatch(r"__must_hold").sub("REPLACED", result) + + self.assertNotIn("__acquires(", result) + self.assertNotIn("__releases(", result) + self.assertNotIn("__must_hold(", result) + + self.assertIn("foo();", result) + + def test_no_macro_remains(self): + """Ensures that unmatched macros are untouched""" + line = "do_something_else();" + result = CMatch(r"__acquires").sub("REPLACED", line) + + self.assertEqual(result, line) + + def test_no_function(self): + """Ensures that no functions will remain untouched""" + line = "something" + result = CMatch(line).sub("REPLACED", line) + + self.assertEqual(result, line) + +# +# Check if the diff is logically equivalent. To simplify, the tests here +# use a single macro name for all replacements. +# + +class TestSubSimple(TestCaseDiff): + """ + Test argument replacements. + + Here, the function name can be anything. So, we picked __attribute__(), + to mimic a macro found at the Kernel, but none of the replacements her + has any relationship with the Kernel usage. + """ + + MACRO = "__attribute__" + + @classmethod + def setUpClass(cls): + """Define a CMatch to be used for all tests""" + cls.matcher = CMatch(cls.MACRO) + + def test_sub_with_capture(self): + """Test all arguments replacement with a single arg""" + line = f"{self.MACRO}(&ctx)\nfoo();" + + result = self.matcher.sub(r"ACQUIRED(\0)", line) + + self.assertLogicallyEqual("ACQUIRED(&ctx)\nfoo();", result) + + def test_sub_zero_placeholder(self): + """Test all arguments replacement with a multiple args""" + line = f"{self.MACRO}(arg1, arg2)\nbar();" + + result = self.matcher.sub(r"REPLACED(\0)", line) + + self.assertLogicallyEqual("REPLACED(arg1, arg2)\nbar();", result) + + def test_sub_single_placeholder(self): + """Single replacement rule for \1""" + line = f"{self.MACRO}(ctx, boo)\nfoo();" + result = self.matcher.sub(r"ACQUIRED(\1)", line) + + self.assertLogicallyEqual("ACQUIRED(ctx)\nfoo();", result) + + def test_sub_multiple_placeholders(self): + """Replacement rule for both \1 and \2""" + line = f"{self.MACRO}(arg1, arg2)\nbar();" + result = self.matcher.sub(r"REPLACE(\1, \2)", line) + + self.assertLogicallyEqual("REPLACE(arg1, arg2)\nbar();", result) + + def test_sub_mixed_placeholders(self): + """Replacement rule for \0, \1 and additional text""" + line = f"{self.MACRO}(foo, bar)\nbaz();" + result = self.matcher.sub(r"ALL(\0) FIRST(\1)", line) + + self.assertLogicallyEqual("ALL(foo, bar) FIRST(foo)\nbaz();", result) + + def test_sub_no_placeholder(self): + """Replacement without placeholders""" + line = f"{self.MACRO}(arg)\nfoo();" + result = self.matcher.sub(r"NO_BACKREFS()", line) + + self.assertLogicallyEqual("NO_BACKREFS()\nfoo();", result) + + def test_sub_count_parameter(self): + """Verify that the algorithm stops after the requested count""" + line = f"{self.MACRO}(a1) x();\n{self.MACRO}(a2) y();" + result = self.matcher.sub(r"ONLY_FIRST(\1) ", line, count=1) + + self.assertLogicallyEqual(f"ONLY_FIRST(a1) x();\n{self.MACRO}(a2) y();", + result) + + def test_strip_multiple_acquires(self): + """Check if spaces between removed delimiters will be dropped""" + line = f"int {self.MACRO}(1) {self.MACRO}(2 ) {self.MACRO}(3) foo;" + result = self.matcher.sub("", line) + + self.assertLogicallyEqual(result, "int foo;") + + def test_rise_early_greedy(self): + line = f"{self.MACRO}(a, b, c, d);" + sub = r"\1, \2+, \3" + + with self.assertRaises(ValueError): + result = self.matcher.sub(sub, line) + + def test_rise_multiple_greedy(self): + line = f"{self.MACRO}(a, b, c, d);" + sub = r"\1, \2+, \3+" + + with self.assertRaises(ValueError): + result = self.matcher.sub(sub, line) + +# +# Test replacements with slashrefs +# + + +class TestSubWithLocalXforms(TestCaseDiff): + """ + Test diferent usecase patterns found at the Kernel. + + Here, replacements using both CMatch and KernRe can be tested, + as it will import the actual replacement rules used by kernel-doc. + """ + + struct_xforms = [ + (CMatch("__attribute__"), ' '), + (CMatch('__aligned'), ' '), + (CMatch('__counted_by'), ' '), + (CMatch('__counted_by_(le|be)'), ' '), + (CMatch('__guarded_by'), ' '), + (CMatch('__pt_guarded_by'), ' '), + + (CMatch('__cacheline_group_(begin|end)'), ''), + + (CMatch('struct_group'), r'\2'), + (CMatch('struct_group_attr'), r'\3'), + (CMatch('struct_group_tagged'), r'struct \1 { \3+ } \2;'), + (CMatch('__struct_group'), r'\4'), + + (CMatch('__ETHTOOL_DECLARE_LINK_MODE_MASK'), r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), + (CMatch('DECLARE_PHY_INTERFACE_MASK',), r'DECLARE_BITMAP(\1, PHY_INTERFACE_MODE_MAX)'), + (CMatch('DECLARE_BITMAP'), r'unsigned long \1[BITS_TO_LONGS(\2)]'), + + (CMatch('DECLARE_HASHTABLE'), r'unsigned long \1[1 << ((\2) - 1)]'), + (CMatch('DECLARE_KFIFO'), r'\2 *\1'), + (CMatch('DECLARE_KFIFO_PTR'), r'\2 *\1'), + (CMatch('(?:__)?DECLARE_FLEX_ARRAY'), r'\1 \2[]'), + (CMatch('DEFINE_DMA_UNMAP_ADDR'), r'dma_addr_t \1'), + (CMatch('DEFINE_DMA_UNMAP_LEN'), r'__u32 \1'), + (CMatch('VIRTIO_DECLARE_FEATURES'), r'union { u64 \1; u64 \1_array[VIRTIO_FEATURES_U64S]; }'), + ] + + function_xforms = [ + (CMatch('__printf'), ""), + (CMatch('__(?:re)?alloc_size'), ""), + (CMatch("__diagnose_as"), ""), + (CMatch("DECL_BUCKET_PARAMS"), r"\1, \2"), + + (CMatch("__cond_acquires"), ""), + (CMatch("__cond_releases"), ""), + (CMatch("__acquires"), ""), + (CMatch("__releases"), ""), + (CMatch("__must_hold"), ""), + (CMatch("__must_not_hold"), ""), + (CMatch("__must_hold_shared"), ""), + (CMatch("__cond_acquires_shared"), ""), + (CMatch("__acquires_shared"), ""), + (CMatch("__releases_shared"), ""), + (CMatch("__attribute__"), ""), + ] + + var_xforms = [ + (CMatch('__guarded_by'), ""), + (CMatch('__pt_guarded_by'), ""), + (CMatch("LIST_HEAD"), r"struct list_head \1"), + ] + + #: Transforms main dictionary used at apply_transforms(). + xforms = { + "struct": struct_xforms, + "func": function_xforms, + "var": var_xforms, + } + + @classmethod + def apply_transforms(cls, xform_type, text): + """ + Mimic the behavior of kdoc_parser.apply_transforms() method. + + For each element of STRUCT_XFORMS, apply apply_transforms. + + There are two parameters: + + - ``xform_type`` + Can be ``func``, ``struct`` or ``var``; + - ``text`` + The text where the sub patterns from CTransforms will be applied. + """ + for search, subst in cls.xforms.get(xform_type): + text = search.sub(subst, text) + + return text.strip() + + cls.matcher = CMatch(r"struct_group[\w\_]*") + + def test_struct_group(self): + """ + Test struct_group using a pattern from + drivers/net/ethernet/asix/ax88796c_main.h. + """ + line = """ + struct tx_pkt_info { + struct_group(tx_overhead, + struct tx_sop_header sop; + struct tx_segment_header seg; + ); + struct tx_eop_header eop; + u16 pkt_len; + u16 seq_num; + }; + """ + expected = """ + struct tx_pkt_info { + struct tx_sop_header sop; + struct tx_segment_header seg; + struct tx_eop_header eop; + u16 pkt_len; + u16 seq_num; + }; + """ + + result = self.apply_transforms("struct", line) + self.assertLogicallyEqual(result, expected) + + def test_struct_group_attr(self): + """ + Test two struct_group_attr using patterns from fs/smb/client/cifspdu.h. + """ + line = """ + typedef struct smb_com_open_rsp { + struct smb_hdr hdr; /* wct = 34 BB */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __u8 OplockLevel; + __u16 Fid; + __le32 CreateAction; + struct_group_attr(common_attributes,, + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le32 FileAttributes; + ); + __le64 AllocationSize; + __le64 EndOfFile; + __le16 FileType; + __le16 DeviceState; + __u8 DirectoryFlag; + __u16 ByteCount; /* bct = 0 */ + } OPEN_RSP; + typedef struct { + struct_group_attr(common_attributes,, + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le32 Attributes; + ); + __u32 Pad1; + __le64 AllocationSize; + __le64 EndOfFile; + __le32 NumberOfLinks; + __u8 DeletePending; + __u8 Directory; + __u16 Pad2; + __le32 EASize; + __le32 FileNameLength; + union { + char __pad; + DECLARE_FLEX_ARRAY(char, FileName); + }; + } FILE_ALL_INFO; /* level 0x107 QPathInfo */ + """ + expected = """ + typedef struct smb_com_open_rsp { + struct smb_hdr hdr; + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __u8 OplockLevel; + __u16 Fid; + __le32 CreateAction; + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le32 FileAttributes; + __le64 AllocationSize; + __le64 EndOfFile; + __le16 FileType; + __le16 DeviceState; + __u8 DirectoryFlag; + __u16 ByteCount; + } OPEN_RSP; + typedef struct { + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le32 Attributes; + __u32 Pad1; + __le64 AllocationSize; + __le64 EndOfFile; + __le32 NumberOfLinks; + __u8 DeletePending; + __u8 Directory; + __u16 Pad2; + __le32 EASize; + __le32 FileNameLength; + union { + char __pad; + char FileName[]; + }; + } FILE_ALL_INFO; + """ + + result = self.apply_transforms("struct", line) + self.assertLogicallyEqual(result, expected) + + def test_raw_struct_group(self): + """ + Test a __struct_group pattern from include/uapi/cxl/features.h. + """ + line = """ + struct cxl_mbox_get_sup_feats_out { + __struct_group(cxl_mbox_get_sup_feats_out_hdr, hdr, /* empty */, + __le16 num_entries; + __le16 supported_feats; + __u8 reserved[4]; + ); + struct cxl_feat_entry ents[] __counted_by_le(num_entries); + } __attribute__ ((__packed__)); + """ + expected = """ + struct cxl_mbox_get_sup_feats_out { + __le16 num_entries; + __le16 supported_feats; + __u8 reserved[4]; + struct cxl_feat_entry ents[]; + }; + """ + + result = self.apply_transforms("struct", line) + self.assertLogicallyEqual(result, expected) + + def test_raw_struct_group_tagged(self): + r""" + Test cxl_regs with struct_group_tagged patterns from drivers/cxl/cxl.h. + + NOTE: + + This one has actually a violation from what kernel-doc would + expect: Kernel-doc regex expects only 3 members, but this is + actually defined as:: + + #define struct_group_tagged(TAG, NAME, MEMBERS...) + + The replace expression there is:: + + struct \1 { \3 } \2; + + but it should be really something like:: + + struct \1 { \3 \4 \5 \6 \7 \8 ... } \2; + + a later fix would be needed to address it. + + """ + line = """ + struct cxl_regs { + struct_group_tagged(cxl_component_regs, component, + void __iomem *hdm_decoder; + void __iomem *ras; + ); + + + /* This is actually a violation: too much commas */ + struct_group_tagged(cxl_device_regs, device_regs, + void __iomem *status, *mbox, *memdev; + ); + + struct_group_tagged(cxl_pmu_regs, pmu_regs, + void __iomem *pmu; + ); + + struct_group_tagged(cxl_rch_regs, rch_regs, + void __iomem *dport_aer; + ); + + struct_group_tagged(cxl_rcd_regs, rcd_regs, + void __iomem *rcd_pcie_cap; + ); + }; + """ + expected = """ + struct cxl_regs { + struct cxl_component_regs { + void __iomem *hdm_decoder; + void __iomem *ras; + } component; + + struct cxl_device_regs { + void __iomem *status, *mbox, *memdev; + } device_regs; + + struct cxl_pmu_regs { + void __iomem *pmu; + } pmu_regs; + + struct cxl_rch_regs { + void __iomem *dport_aer; + } rch_regs; + + struct cxl_rcd_regs { + void __iomem *rcd_pcie_cap; + } rcd_regs; + }; + """ + + result = self.apply_transforms("struct", line) + self.assertLogicallyEqual(result, expected) + + def test_struct_group_tagged_with_private(self): + """ + Replace struct_group_tagged with private, using the same regex + for the replacement as what happens in xforms_lists.py. + + As the private removal happens outside NestedGroup class, we manually + dropped the remaining part of the struct, to simulate what happens + at kdoc_parser. + + Taken from include/net/page_pool/types.h + """ + line = """ + struct page_pool_params { + struct_group_tagged(page_pool_params_slow, slow, + struct net_device *netdev; + unsigned int queue_idx; + unsigned int flags; + /* private: only under "slow" struct */ + unsigned int ignored; + ); + /* Struct below shall not be ignored */ + struct_group_tagged(page_pool_params_fast, fast, + unsigned int order; + unsigned int pool_size; + int nid; + struct device *dev; + struct napi_struct *napi; + enum dma_data_direction dma_dir; + unsigned int max_len; + unsigned int offset; + ); + }; + """ + expected = """ + struct page_pool_params { + struct page_pool_params_slow { + struct net_device *netdev; + unsigned int queue_idx; + unsigned int flags; + } slow; + struct page_pool_params_fast { + unsigned int order; + unsigned int pool_size; + int nid; + struct device *dev; + struct napi_struct *napi; + enum dma_data_direction dma_dir; + unsigned int max_len; + unsigned int offset; + } fast; + }; + """ + + result = self.apply_transforms("struct", line) + self.assertLogicallyEqual(result, expected) + + def test_struct_kcov(self): + """ + """ + line = """ + struct kcov { + refcount_t refcount; + spinlock_t lock; + enum kcov_mode mode __guarded_by(&lock); + unsigned int size __guarded_by(&lock); + void *area __guarded_by(&lock); + struct task_struct *t __guarded_by(&lock); + bool remote; + unsigned int remote_size; + int sequence; + }; + """ + expected = """ + """ + + result = self.apply_transforms("struct", line) + self.assertLogicallyEqual(result, expected) + + + def test_struct_kcov(self): + """ + Test a struct from kernel/kcov.c. + """ + line = """ + struct kcov { + refcount_t refcount; + spinlock_t lock; + enum kcov_mode mode __guarded_by(&lock); + unsigned int size __guarded_by(&lock); + void *area __guarded_by(&lock); + struct task_struct *t __guarded_by(&lock); + bool remote; + unsigned int remote_size; + int sequence; + }; + """ + expected = """ + struct kcov { + refcount_t refcount; + spinlock_t lock; + enum kcov_mode mode; + unsigned int size; + void *area; + struct task_struct *t; + bool remote; + unsigned int remote_size; + int sequence; + }; + """ + + result = self.apply_transforms("struct", line) + self.assertLogicallyEqual(result, expected) + + def test_vars_stackdepot(self): + """ + Test guarded_by on vars from lib/stackdepot.c. + """ + line = """ + size_t pool_offset __guarded_by(&pool_lock) = DEPOT_POOL_SIZE; + __guarded_by(&pool_lock) LIST_HEAD(free_stacks); + void **stack_pools __pt_guarded_by(&pool_lock); + """ + expected = """ + size_t pool_offset = DEPOT_POOL_SIZE; + struct list_head free_stacks; + void **stack_pools; + """ + + result = self.apply_transforms("var", line) + self.assertLogicallyEqual(result, expected) + + def test_functions_with_acquires_and_releases(self): + """ + Test guarded_by on vars from lib/stackdepot.c. + """ + line = """ + bool prepare_report_consumer(unsigned long *flags, + const struct access_info *ai, + struct other_info *other_info) \ + __cond_acquires(true, &report_lock); + + int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c) \ + __cond_acquires(0, RCU_BH); + + bool undo_report_consumer(unsigned long *flags, + const struct access_info *ai, + struct other_info *other_info) \ + __cond_releases(true, &report_lock); + + void debugfs_enter_cancellation(struct file *file, + struct debugfs_cancellation *c) \ + __acquires(cancellation); + + void debugfs_leave_cancellation(struct file *file, + struct debugfs_cancellation *c) \ + __releases(cancellation); + + acpi_cpu_flags acpi_os_acquire_lock(acpi_spinlock lockp) \ + __acquires(lockp); + + void acpi_os_release_lock(acpi_spinlock lockp, + acpi_cpu_flags not_used) \ + __releases(lockp) + """ + expected = """ + bool prepare_report_consumer(unsigned long *flags, + const struct access_info *ai, + struct other_info *other_info); + + int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c); + + bool undo_report_consumer(unsigned long *flags, + const struct access_info *ai, + struct other_info *other_info); + + void debugfs_enter_cancellation(struct file *file, + struct debugfs_cancellation *c); + + void debugfs_leave_cancellation(struct file *file, + struct debugfs_cancellation *c); + + acpi_cpu_flags acpi_os_acquire_lock(acpi_spinlock lockp); + + void acpi_os_release_lock(acpi_spinlock lockp, + acpi_cpu_flags not_used) + """ + + result = self.apply_transforms("func", line) + self.assertLogicallyEqual(result, expected) + +# +# Run all tests +# +if __name__ == "__main__": + run_unittest(__file__) diff --git a/tools/unittests/test_kdoc_parser.py b/tools/unittests/test_kdoc_parser.py new file mode 100755 index 000000000000..c4a76ed13dbc --- /dev/null +++ b/tools/unittests/test_kdoc_parser.py @@ -0,0 +1,560 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2026: Mauro Carvalho Chehab <mchehab@kernel.org>. +# +# pylint: disable=C0200,C0413,W0102,R0914 + +""" +Unit tests for kernel-doc parser. +""" + +import logging +import os +import re +import shlex +import sys +import unittest + +from textwrap import dedent +from unittest.mock import patch, MagicMock, mock_open + +import yaml + +SRC_DIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(SRC_DIR, "../lib/python")) + +from kdoc.kdoc_files import KdocConfig +from kdoc.kdoc_item import KdocItem +from kdoc.kdoc_parser import KernelDoc +from kdoc.kdoc_output import RestFormat, ManFormat + +from kdoc.xforms_lists import CTransforms + +from unittest_helper import TestUnits + + +# +# Test file +# +TEST_FILE = os.path.join(SRC_DIR, "kdoc-test.yaml") + +env = { + "yaml_file": TEST_FILE +} + +# +# Ancillary logic to clean whitespaces +# +#: Regex to help cleaning whitespaces +RE_WHITESPC = re.compile(r"[ \t]++") +RE_BEGINSPC = re.compile(r"^\s+", re.MULTILINE) +RE_ENDSPC = re.compile(r"\s+$", re.MULTILINE) + +def clean_whitespc(val, relax_whitespace=False): + """ + Cleanup whitespaces to avoid false positives. + + By default, strip only bein/end whitespaces, but, when relax_whitespace + is true, also replace multiple whitespaces in the middle. + """ + + if isinstance(val, str): + val = val.strip() + if relax_whitespace: + val = RE_WHITESPC.sub(" ", val) + val = RE_BEGINSPC.sub("", val) + val = RE_ENDSPC.sub("", val) + elif isinstance(val, list): + val = [clean_whitespc(item, relax_whitespace) for item in val] + elif isinstance(val, dict): + val = {k: clean_whitespc(v, relax_whitespace) for k, v in val.items()} + return val + +# +# Helper classes to help mocking with logger and config +# +class MockLogging(logging.Handler): + """ + Simple class to store everything on a list + """ + + def __init__(self, level=logging.NOTSET): + super().__init__(level) + self.messages = [] + self.formatter = logging.Formatter() + + def emit(self, record: logging.LogRecord) -> None: + """ + Append a formatted record to self.messages. + """ + try: + # The `format` method uses the handler's formatter. + message = self.format(record) + self.messages.append(message) + except Exception: + self.handleError(record) + +class MockKdocConfig(KdocConfig): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.log = logging.getLogger(__file__) + self.handler = MockLogging() + self.log.addHandler(self.handler) + + def warning(self, msg): + """Ancillary routine to output a warning and increment error count.""" + + self.log.warning(msg) + +# +# Helper class to generate KdocItem and validate its contents +# +# TODO: check self.config.handler.messages content +# +class GenerateKdocItem(unittest.TestCase): + """ + Base class to run KernelDoc parser class + """ + + DEFAULT = vars(KdocItem("", "", "", 0)) + + config = MockKdocConfig() + xforms = CTransforms() + + def setUp(self): + self.maxDiff = None + + def run_test(self, source, __expected_list, exports={}, fname="test.c", + relax_whitespace=False): + """ + Stores expected values and patch the test to use source as + a "file" input. + """ + debug_level = int(os.getenv("VERBOSE", "0")) + source = dedent(source) + + # Ensure that default values will be there + expected_list = [] + for e in __expected_list: + if not isinstance(e, dict): + e = vars(e) + + new_e = self.DEFAULT.copy() + new_e["fname"] = fname + for key, value in e.items(): + new_e[key] = value + + expected_list.append(new_e) + + patcher = patch('builtins.open', + new_callable=mock_open, read_data=source) + + kernel_doc = KernelDoc(self.config, fname, self.xforms) + + with patcher: + export_table, entries = kernel_doc.parse_kdoc() + + self.assertEqual(export_table, exports) + self.assertEqual(len(entries), len(expected_list)) + + for i in range(0, len(entries)): + + entry = entries[i] + expected = expected_list[i] + self.assertNotEqual(expected, None) + self.assertNotEqual(expected, {}) + self.assertIsInstance(entry, KdocItem) + + d = vars(entry) + + other_stuff = d.get("other_stuff", {}) + if "source" in other_stuff: + del other_stuff["source"] + + for key, value in expected.items(): + if key == "other_stuff": + if "source" in value: + del value["source"] + + result = clean_whitespc(d[key], relax_whitespace) + value = clean_whitespc(value, relax_whitespace) + + if debug_level > 1: + sys.stderr.write(f"{key}: assert('{result}' == '{value}')\n") + + self.assertEqual(result, value, msg=f"at {key}") + +# +# Ancillary function that replicates kdoc_files way to generate output +# +def cleanup_timestamp(text): + lines = text.split("\n") + + for i, line in enumerate(lines): + if not line.startswith('.TH'): + continue + + parts = shlex.split(line) + if len(parts) > 3: + parts[3] = "" + + lines[i] = " ".join(parts) + + + return "\n".join(lines) + +def gen_output(fname, out_style, symbols, expected, + config=None, relax_whitespace=False): + """ + Use the output class to return an output content from KdocItem symbols. + """ + + if not config: + config = MockKdocConfig() + + out_style.set_config(config) + + msg = out_style.output_symbols(fname, symbols) + + result = clean_whitespc(msg, relax_whitespace) + result = cleanup_timestamp(result) + + expected = clean_whitespc(expected, relax_whitespace) + expected = cleanup_timestamp(expected) + + return result, expected + +# +# Classes to be used by dynamic test generation from YAML +# +class CToKdocItem(GenerateKdocItem): + def setUp(self): + self.maxDiff = None + + def run_parser_test(self, source, symbols, exports, fname): + if isinstance(symbols, dict): + symbols = [symbols] + + if isinstance(exports, str): + exports=set([exports]) + elif isinstance(exports, list): + exports=set(exports) + + self.run_test(source, symbols, exports=exports, + fname=fname, relax_whitespace=True) + +class KdocItemToMan(unittest.TestCase): + out_style = ManFormat() + + def setUp(self): + self.maxDiff = None + + def run_out_test(self, fname, symbols, expected): + """ + Generate output using out_style, + """ + result, expected = gen_output(fname, self.out_style, + symbols, expected) + + self.assertEqual(result, expected) + +class KdocItemToRest(unittest.TestCase): + out_style = RestFormat() + + def setUp(self): + self.maxDiff = None + + def run_out_test(self, fname, symbols, expected): + """ + Generate output using out_style, + """ + result, expected = gen_output(fname, self.out_style, symbols, + expected, relax_whitespace=True) + + self.assertEqual(result, expected) + + +class CToMan(unittest.TestCase): + out_style = ManFormat() + config = MockKdocConfig() + xforms = CTransforms() + + def setUp(self): + self.maxDiff = None + + def run_out_test(self, fname, source, expected): + """ + Generate output using out_style, + """ + patcher = patch('builtins.open', + new_callable=mock_open, read_data=source) + + kernel_doc = KernelDoc(self.config, fname, self.xforms) + + with patcher: + export_table, entries = kernel_doc.parse_kdoc() + + result, expected = gen_output(fname, self.out_style, + entries, expected, config=self.config) + + self.assertEqual(result, expected) + + +class CToRest(unittest.TestCase): + out_style = RestFormat() + config = MockKdocConfig() + xforms = CTransforms() + + def setUp(self): + self.maxDiff = None + + def run_out_test(self, fname, source, expected): + """ + Generate output using out_style, + """ + patcher = patch('builtins.open', + new_callable=mock_open, read_data=source) + + kernel_doc = KernelDoc(self.config, fname, self.xforms) + + with patcher: + export_table, entries = kernel_doc.parse_kdoc() + + result, expected = gen_output(fname, self.out_style, entries, + expected, relax_whitespace=True, + config=self.config) + + self.assertEqual(result, expected) + + +# +# Selftest class +# +class TestSelfValidate(GenerateKdocItem): + """ + Tests to check if logic inside GenerateKdocItem.run_test() is working. + """ + + SOURCE = """ + /** + * function3: Exported function + * @arg1: @arg1 does nothing + * + * Does nothing + * + * return: + * always return 0. + */ + int function3(char *arg1) { return 0; }; + EXPORT_SYMBOL(function3); + """ + + EXPECTED = [{ + 'name': 'function3', + 'type': 'function', + 'declaration_start_line': 2, + + 'sections_start_lines': { + 'Description': 4, + 'Return': 7, + }, + 'sections': { + 'Description': 'Does nothing\n\n', + 'Return': '\nalways return 0.\n' + }, + + 'sections_start_lines': { + 'Description': 4, + 'Return': 7, + }, + + 'parameterdescs': {'arg1': '@arg1 does nothing\n'}, + 'parameterlist': ['arg1'], + 'parameterdesc_start_lines': {'arg1': 3}, + 'parametertypes': {'arg1': 'char *arg1'}, + + 'other_stuff': { + 'func_macro': False, + 'functiontype': 'int', + 'purpose': 'Exported function', + 'typedef': False + }, + }] + + EXPORTS = {"function3"} + + def test_parse_pass(self): + """ + Test if export_symbol is properly handled. + """ + self.run_test(self.SOURCE, self.EXPECTED, self.EXPORTS) + + @unittest.expectedFailure + def test_no_exports(self): + """ + Test if export_symbol is properly handled. + """ + self.run_test(self.SOURCE, [], {}) + + @unittest.expectedFailure + def test_with_empty_expected(self): + """ + Test if export_symbol is properly handled. + """ + self.run_test(self.SOURCE, [], self.EXPORTS) + + @unittest.expectedFailure + def test_with_unfilled_expected(self): + """ + Test if export_symbol is properly handled. + """ + self.run_test(self.SOURCE, [{}], self.EXPORTS) + + @unittest.expectedFailure + def test_with_default_expected(self): + """ + Test if export_symbol is properly handled. + """ + self.run_test(self.SOURCE, [self.DEFAULT.copy()], self.EXPORTS) + +# +# Class and logic to create dynamic tests from YAML +# + +class KernelDocDynamicTests(): + """ + Dynamically create a set of tests from a YAML file. + """ + + @classmethod + def create_parser_test(cls, name, fname, source, symbols, exports): + """ + Return a function that will be attached to the test class. + """ + def test_method(self): + """Lambda-like function to run tests with provided vars""" + self.run_parser_test(source, symbols, exports, fname) + + test_method.__name__ = f"test_gen_{name}" + + setattr(CToKdocItem, test_method.__name__, test_method) + + @classmethod + def create_out_test(cls, name, fname, symbols, out_type, data): + """ + Return a function that will be attached to the test class. + """ + def test_method(self): + """Lambda-like function to run tests with provided vars""" + self.run_out_test(fname, symbols, data) + + test_method.__name__ = f"test_{out_type}_{name}" + + if out_type == "man": + setattr(KdocItemToMan, test_method.__name__, test_method) + else: + setattr(KdocItemToRest, test_method.__name__, test_method) + + @classmethod + def create_src2out_test(cls, name, fname, source, out_type, data): + """ + Return a function that will be attached to the test class. + """ + def test_method(self): + """Lambda-like function to run tests with provided vars""" + self.run_out_test(fname, source, data) + + test_method.__name__ = f"test_{out_type}_{name}" + + if out_type == "man": + setattr(CToMan, test_method.__name__, test_method) + else: + setattr(CToRest, test_method.__name__, test_method) + + @classmethod + def create_tests(cls): + """ + Iterate over all scenarios and add a method to the class for each. + + The logic in this function assumes a valid test that are compliant + with kdoc-test-schema.yaml. There is an unit test to check that. + As such, it picks mandatory values directly, and uses get() for the + optional ones. + """ + + test_file = os.environ.get("yaml_file", TEST_FILE) + + with open(test_file, encoding="utf-8") as fp: + testset = yaml.safe_load(fp) + + tests = testset["tests"] + + for idx, test in enumerate(tests): + name = test["name"] + fname = test["fname"] + source = test["source"] + expected_list = test["expected"] + + exports = test.get("exports", []) + + # + # The logic below allows setting up to 5 types of test: + # 1. from source to kdoc_item: test KernelDoc class; + # 2. from kdoc_item to man: test ManOutput class; + # 3. from kdoc_item to rst: test RestOutput class; + # 4. from source to man without checking expected KdocItem; + # 5. from source to rst without checking expected KdocItem. + # + for expected in expected_list: + kdoc_item = expected.get("kdoc_item") + man = expected.get("man", []) + rst = expected.get("rst", []) + + if kdoc_item: + if isinstance(kdoc_item, dict): + kdoc_item = [kdoc_item] + + symbols = [] + + for arg in kdoc_item: + arg["fname"] = fname + arg["start_line"] = 1 + + symbols.append(KdocItem.from_dict(arg)) + + if source: + cls.create_parser_test(name, fname, source, + symbols, exports) + + if man: + cls.create_out_test(name, fname, symbols, "man", man) + + if rst: + cls.create_out_test(name, fname, symbols, "rst", rst) + + elif source: + if man: + cls.create_src2out_test(name, fname, source, "man", man) + + if rst: + cls.create_src2out_test(name, fname, source, "rst", rst) + +KernelDocDynamicTests.create_tests() + +# +# Run all tests +# +if __name__ == "__main__": + runner = TestUnits() + parser = runner.parse_args() + parser.add_argument("-y", "--yaml-file", "--yaml", + help='Name of the yaml file to load') + + args = parser.parse_args() + + if args.yaml_file: + env["yaml_file"] = os.path.expanduser(args.yaml_file) + + # Run tests with customized arguments + runner.run(__file__, parser=parser, args=args, env=env) diff --git a/tools/unittests/test_kdoc_test_schema.py b/tools/unittests/test_kdoc_test_schema.py new file mode 100755 index 000000000000..9eceeba00440 --- /dev/null +++ b/tools/unittests/test_kdoc_test_schema.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Unit‑test driver for kernel‑doc YAML tests. + +Two kinds of tests are defined: + +* **Schema‑validation tests** – if ``jsonschema`` is available, the + YAML files in this directory are validated against the JSON‑Schema + described in ``kdoc-test-schema.yaml``. When the library is not + present, a warning is emitted and the validation step is simply + skipped – the dynamic kernel‑doc tests still run. + +* **Kernel‑doc tests** – dynamically generate one test method per + scenario in ``kdoc-test.yaml``. Each method simply forwards + the data to ``self.run_test`` – you only need to implement that + helper in your own code. + +File names are kept as module‑level constants so that the +implementation stays completely independent of ``pathlib``. +""" + +import os +import sys +import warnings +import yaml +import unittest +from typing import Any, Dict, List + +SRC_DIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(SRC_DIR, "../lib/python")) + +from unittest_helper import run_unittest + + +# +# Files to read +# +BASE = os.path.realpath(os.path.dirname(__file__)) + +SCHEMA_FILE = os.path.join(BASE, "kdoc-test-schema.yaml") +TEST_FILE = os.path.join(BASE, "kdoc-test.yaml") + +# +# Schema‑validation test +# +class TestYAMLSchemaValidation(unittest.TestCase): + """ + Checks if TEST_FILE matches SCHEMA_FILE. + """ + + @classmethod + def setUpClass(cls): + """ + Import jsonschema if available. + """ + + try: + from jsonschema import Draft7Validator + except ImportError: + print("Warning: jsonschema package not available. Skipping schema validation") + cls.validator = None + return + + with open(SCHEMA_FILE, encoding="utf-8") as fp: + cls.schema = yaml.safe_load(fp) + + cls.validator = Draft7Validator(cls.schema) + + def test_kdoc_test_yaml_followsschema(self): + """ + Run jsonschema validation if the validator is available. + If not, emit a warning and return without failing. + """ + if self.validator is None: + return + + with open(TEST_FILE, encoding="utf-8") as fp: + data = yaml.safe_load(fp) + + errors = self.validator.iter_errors(data) + + msgs = [] + for error in errors: + msgs.append(error.message) + + if msgs: + self.fail("Schema validation failed:\n\t" + "\n\t".join(msgs)) + +# -------------------------------------------------------------------- +# Entry point +# -------------------------------------------------------------------- +if __name__ == "__main__": + run_unittest(__file__) diff --git a/tools/unittests/test_tokenizer.py b/tools/unittests/test_tokenizer.py new file mode 100755 index 000000000000..d1f3c565b9cf --- /dev/null +++ b/tools/unittests/test_tokenizer.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 + +""" +Unit tests for struct/union member extractor class. +""" + + +import os +import re +import unittest +import sys + +from unittest.mock import MagicMock + +SRC_DIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(SRC_DIR, "../lib/python")) + +from kdoc.c_lex import CToken, CTokenizer +from unittest_helper import run_unittest + +# +# List of tests. +# +# The code will dynamically generate one test for each key on this dictionary. +# +def tokens_to_list(tokens): + tuples = [] + + for tok in tokens: + if tok.kind == CToken.SPACE: + continue + + tuples += [(tok.kind, tok.value, tok.level)] + + return tuples + + +def make_tokenizer_test(name, data): + """ + Create a test named ``name`` using parameters given by ``data`` dict. + """ + + def test(self): + """In-lined lambda-like function to run the test""" + + # + # Check if logger is working + # + if "log_msg" in data: + with self.assertLogs() as cm: + tokenizer = CTokenizer(data["source"]) + + msg_found = False + for result in cm.output: + if data["log_msg"] in result: + msg_found = True + + self.assertTrue(msg_found, f"Missing log {data['log_msg']}") + + return + + # + # Check if tokenizer is producing expected results + # + tokens = CTokenizer(data["source"]).tokens + + result = tokens_to_list(tokens) + expected = tokens_to_list(data["expected"]) + + self.assertEqual(result, expected, msg=f"{name}") + + return test + +#: Tokenizer tests. +TESTS_TOKENIZER = { + "__run__": make_tokenizer_test, + + "basic_tokens": { + "source": """ + int a; // comment + float b = 1.23; + """, + "expected": [ + CToken(CToken.NAME, "int"), + CToken(CToken.NAME, "a"), + CToken(CToken.ENDSTMT, ";"), + CToken(CToken.COMMENT, "// comment"), + CToken(CToken.NAME, "float"), + CToken(CToken.NAME, "b"), + CToken(CToken.OP, "="), + CToken(CToken.NUMBER, "1.23"), + CToken(CToken.ENDSTMT, ";"), + ], + }, + + "depth_counters": { + "source": """ + struct X { + int arr[10]; + func(a[0], (b + c)); + } + """, + "expected": [ + CToken(CToken.STRUCT, "struct"), + CToken(CToken.NAME, "X"), + CToken(CToken.BEGIN, "{", brace_level=1), + + CToken(CToken.NAME, "int", brace_level=1), + CToken(CToken.NAME, "arr", brace_level=1), + CToken(CToken.BEGIN, "[", brace_level=1, bracket_level=1), + CToken(CToken.NUMBER, "10", brace_level=1, bracket_level=1), + CToken(CToken.END, "]", brace_level=1), + CToken(CToken.ENDSTMT, ";", brace_level=1), + CToken(CToken.NAME, "func", brace_level=1), + CToken(CToken.BEGIN, "(", brace_level=1, paren_level=1), + CToken(CToken.NAME, "a", brace_level=1, paren_level=1), + CToken(CToken.BEGIN, "[", brace_level=1, paren_level=1, bracket_level=1), + CToken(CToken.NUMBER, "0", brace_level=1, paren_level=1, bracket_level=1), + CToken(CToken.END, "]", brace_level=1, paren_level=1), + CToken(CToken.PUNC, ",", brace_level=1, paren_level=1), + CToken(CToken.BEGIN, "(", brace_level=1, paren_level=2), + CToken(CToken.NAME, "b", brace_level=1, paren_level=2), + CToken(CToken.OP, "+", brace_level=1, paren_level=2), + CToken(CToken.NAME, "c", brace_level=1, paren_level=2), + CToken(CToken.END, ")", brace_level=1, paren_level=1), + CToken(CToken.END, ")", brace_level=1), + CToken(CToken.ENDSTMT, ";", brace_level=1), + CToken(CToken.END, "}"), + ], + }, + + "mismatch_error": { + "source": "int a$ = 5;", # $ is illegal + "log_msg": "Unexpected token", + }, +} + +def make_private_test(name, data): + """ + Create a test named ``name`` using parameters given by ``data`` dict. + """ + + def test(self): + """In-lined lambda-like function to run the test""" + tokens = CTokenizer(data["source"]) + result = str(tokens) + + # + # Avoid whitespace false positives + # + result = re.sub(r"\s++", " ", result).strip() + expected = re.sub(r"\s++", " ", data["trimmed"]).strip() + + msg = f"failed when parsing this source:\n{data['source']}" + self.assertEqual(result, expected, msg=msg) + + return test + +#: Tests to check if CTokenizer is handling properly public/private comments. +TESTS_PRIVATE = { + # + # Simplest case: no private. Ensure that trimming won't affect struct + # + "__run__": make_private_test, + "no private": { + "source": """ + struct foo { + int a; + int b; + int c; + }; + """, + "trimmed": """ + struct foo { + int a; + int b; + int c; + }; + """, + }, + + # + # Play "by the books" by always having a public in place + # + + "balanced_private": { + "source": """ + struct foo { + int a; + /* private: */ + int b; + /* public: */ + int c; + }; + """, + "trimmed": """ + struct foo { + int a; + int c; + }; + """, + }, + + "balanced_non_greddy_private": { + "source": """ + struct foo { + int a; + /* private: */ + int b; + /* public: */ + int c; + /* private: */ + int d; + /* public: */ + int e; + + }; + """, + "trimmed": """ + struct foo { + int a; + int c; + int e; + }; + """, + }, + + "balanced_inner_private": { + "source": """ + struct foo { + struct { + int a; + /* private: ignore below */ + int b; + /* public: but this should not be ignored */ + }; + int b; + }; + """, + "trimmed": """ + struct foo { + struct { + int a; + }; + int b; + }; + """, + }, + + # + # Test what happens if there's no public after private place + # + + "unbalanced_private": { + "source": """ + struct foo { + int a; + /* private: */ + int b; + int c; + }; + """, + "trimmed": """ + struct foo { + int a; + }; + """, + }, + + "unbalanced_inner_private": { + "source": """ + struct foo { + struct { + int a; + /* private: ignore below */ + int b; + /* but this should not be ignored */ + }; + int b; + }; + """, + "trimmed": """ + struct foo { + struct { + int a; + }; + int b; + }; + """, + }, + + "unbalanced_struct_group_tagged_with_private": { + "source": """ + struct page_pool_params { + struct_group_tagged(page_pool_params_fast, fast, + unsigned int order; + unsigned int pool_size; + int nid; + struct device *dev; + struct napi_struct *napi; + enum dma_data_direction dma_dir; + unsigned int max_len; + unsigned int offset; + }; + struct_group_tagged(page_pool_params_slow, slow, + struct net_device *netdev; + unsigned int queue_idx; + unsigned int flags; + /* private: used by test code only */ + void (*init_callback)(netmem_ref netmem, void *arg); + void *init_arg; + }; + }; + """, + "trimmed": """ + struct page_pool_params { + struct_group_tagged(page_pool_params_fast, fast, + unsigned int order; + unsigned int pool_size; + int nid; + struct device *dev; + struct napi_struct *napi; + enum dma_data_direction dma_dir; + unsigned int max_len; + unsigned int offset; + }; + struct_group_tagged(page_pool_params_slow, slow, + struct net_device *netdev; + unsigned int queue_idx; + unsigned int flags; + }; + }; + """, + }, + + "unbalanced_two_struct_group_tagged_first_with_private": { + "source": """ + struct page_pool_params { + struct_group_tagged(page_pool_params_slow, slow, + struct net_device *netdev; + unsigned int queue_idx; + unsigned int flags; + /* private: used by test code only */ + void (*init_callback)(netmem_ref netmem, void *arg); + void *init_arg; + }; + struct_group_tagged(page_pool_params_fast, fast, + unsigned int order; + unsigned int pool_size; + int nid; + struct device *dev; + struct napi_struct *napi; + enum dma_data_direction dma_dir; + unsigned int max_len; + unsigned int offset; + }; + }; + """, + "trimmed": """ + struct page_pool_params { + struct_group_tagged(page_pool_params_slow, slow, + struct net_device *netdev; + unsigned int queue_idx; + unsigned int flags; + }; + struct_group_tagged(page_pool_params_fast, fast, + unsigned int order; + unsigned int pool_size; + int nid; + struct device *dev; + struct napi_struct *napi; + enum dma_data_direction dma_dir; + unsigned int max_len; + unsigned int offset; + }; + }; + """, + }, + "unbalanced_without_end_of_line": { + "source": """ \ + struct page_pool_params { \ + struct_group_tagged(page_pool_params_slow, slow, \ + struct net_device *netdev; \ + unsigned int queue_idx; \ + unsigned int flags; + /* private: used by test code only */ + void (*init_callback)(netmem_ref netmem, void *arg); \ + void *init_arg; \ + }; \ + struct_group_tagged(page_pool_params_fast, fast, \ + unsigned int order; \ + unsigned int pool_size; \ + int nid; \ + struct device *dev; \ + struct napi_struct *napi; \ + enum dma_data_direction dma_dir; \ + unsigned int max_len; \ + unsigned int offset; \ + }; \ + }; + """, + "trimmed": """ + struct page_pool_params { + struct_group_tagged(page_pool_params_slow, slow, + struct net_device *netdev; + unsigned int queue_idx; + unsigned int flags; + }; + struct_group_tagged(page_pool_params_fast, fast, + unsigned int order; + unsigned int pool_size; + int nid; + struct device *dev; + struct napi_struct *napi; + enum dma_data_direction dma_dir; + unsigned int max_len; + unsigned int offset; + }; + }; + """, + }, +} + +#: Dict containing all test groups fror CTokenizer +TESTS = { + "TestPublicPrivate": TESTS_PRIVATE, + "TestTokenizer": TESTS_TOKENIZER, +} + +def setUp(self): + self.maxDiff = None + +def build_test_class(group_name, table): + """ + Dynamically creates a class instance using type() as a generator + for a new class derivated from unittest.TestCase. + + We're opting to do it inside a function to avoid the risk of + changing the globals() dictionary. + """ + + class_dict = { + "setUp": setUp + } + + run = table["__run__"] + + for test_name, data in table.items(): + if test_name == "__run__": + continue + + class_dict[f"test_{test_name}"] = run(test_name, data) + + cls = type(group_name, (unittest.TestCase,), class_dict) + + return cls.__name__, cls + +# +# Create classes and add them to the global dictionary +# +for group, table in TESTS.items(): + t = build_test_class(group, table) + globals()[t[0]] = t[1] + +# +# main +# +if __name__ == "__main__": + run_unittest(__file__) |
