augment_headers.py

#!/usr/bin/env python3

# Script to augment the C headers with autogenerated content, including:
# - declarations for any aliases of existing function and data declarations in
#   the C headers, as specified in their corresponding symbol table entries.
# - docstrings for function and data declarations in the C headers, generated
#   from their corresponding symbol table entries, if present.
#
# By default, this script edits header files in-place. It's recommended that
# you commit any header file changes before running this script.

from abc import ABC
import argparse
import fnmatch
import os
import re
import subprocess
import textwrap
from typing import Dict, Iterator, List, Optional, Set, Tuple
import yaml

from symbol_check import (
    FunctionList,
    DataList,
    HeaderSymbolList,
    ROOT_DIR,
)


class Formatter(ABC):
    def format_docstring(self, text: str) -> str:
        raise NotImplementedError

    def format_file(self, filename: str):
        raise NotImplementedError

    def sanitize_comment(self, comment: str) -> str:
        # Break up any end-of-comment delimiters in the raw comment text
        return comment.replace("*/", "* /")


class ClangFormatFormatter(Formatter):
    def __init__(self):
        # Check for clang-format in the runtime environment
        subprocess.run(["clang-format", "--version"], capture_output=True, check=True)

    def format_docstring(self, text: str) -> str:
        docstring = "/**\n"
        for line in text.splitlines():
            docstring += f" * {self.sanitize_comment(line)}\n"
        docstring += " */\n"
        return docstring

    def format_file(self, filename: str):
        subprocess.run(["clang-format", "-i", filename], check=True)


class TextWrapFormatter(Formatter):
    """Fallback formatter if clang-format isn't found in the environment"""

    def __init__(self):
        self.linewidth = 100
        try:
            clang_format_file = os.path.join(ROOT_DIR, "headers", ".clang-format")
            with open(clang_format_file, "r") as f:
                clang_format_options = yaml.safe_load(f)
            self.linewidth = int(clang_format_options["ColumnLimit"])
        except Exception:
            pass

    def format_docstring(self, text: str) -> str:
        prefix = " * "
        # Text wrapping needs to be done line-by-line to properly preserve
        # empty lines
        docstring = "/**\n"
        for line in text.splitlines():
            wrapped_lines = textwrap.fill(
                self.sanitize_comment(line),
                self.linewidth,
                initial_indent=prefix,
                subsequent_indent=prefix,
                replace_whitespace=False,
                break_long_words=False,
            )
            # the initial_indent and subsequent_indent parameters don't work
            # with empty lines, so add the bare prefix directly in this case
            docstring += wrapped_lines if wrapped_lines else prefix.rstrip()
            docstring += "\n"
        docstring += " */\n"
        return docstring

    def format_file(self, filename: str):
        return


class HeaderAugmenter:
    DOCSTRING_PREAMBLE_LINE = "// THIS DOCSTRING WAS GENERATED AUTOMATICALLY\n"
    CPP_COMMENT_RE = re.compile(r"\s*//")
    INCLUDE_GUARD_RE = re.compile(r"\s*#define HEADERS_[A-Z0-9_]+_H_?")

    def __init__(
        self,
        symbol_list: HeaderSymbolList,
        *,
        extension: str = "",
        formatter: Optional[Formatter] = None,
    ):
        self.slist = symbol_list
        self.in_extension = ""
        self.out_extension = extension
        self.formatter = formatter if formatter is not None else TextWrapFormatter()
        # Parse symbol aliases and descriptions from the YAML symbol table
        with open(symbol_list.symbol_file, "r") as f:
            symbol_table = yaml.safe_load(f)
        self.symbol_aliases: Dict[str, List[str]] = {}
        self.symbol_descriptions: Dict[str, str] = {}
        for block in symbol_table.values():
            for symbol in block[symbol_list.SYMBOL_LIST_KEY]:
                aliases = symbol.get("aliases", [])
                if aliases:
                    self.symbol_aliases[symbol["name"]] = aliases
                if "description" in symbol:
                    for name in [symbol["name"]] + aliases:
                        self.symbol_descriptions[name] = symbol["description"]

    def input_header_file(self):
        return self.slist.header_file + self.in_extension

    def output_header_file(self):
        return self.slist.header_file + self.out_extension

    def commit_header_file(self, format: bool = True):
        if format:
            self.formatter.format_file(self.output_header_file())
        self.in_extension = self.out_extension

    def load_symbol_names(self) -> Set[str]:
        return set(self.slist.names_from_c_header(self.input_header_file()))

    @staticmethod
    def _input_header_lines(lines) -> Iterator[Tuple[str, bool, bool, bool, bool]]:
        in_docstring = False
        in_c_style_comment = False
        in_directive = False
        is_deprecated_macro = False
        for line in lines:
            in_cpp_style_comment = False
            if line == HeaderAugmenter.DOCSTRING_PREAMBLE_LINE:
                in_docstring = True
                in_cpp_style_comment = True
                in_c_style_comment = False
            else:
                if HeaderAugmenter.CPP_COMMENT_RE.match(line):
                    in_cpp_style_comment = True
                elif not in_c_style_comment and "/*" in line:
                    in_c_style_comment = True

            in_comment = in_c_style_comment or in_cpp_style_comment
            if not in_comment:
                in_docstring = False

            in_directive = not in_comment and line.lstrip().startswith("#")
            is_deprecated_macro = not in_comment and line.lstrip().startswith("DEPRECATED(")

            yield line, in_comment, in_docstring, in_directive, is_deprecated_macro

            if in_c_style_comment and "*/" in line:
                in_c_style_comment = False

    def add_aliases(self, mark_as_deprecated: bool) -> int:
        add_count = 0
        symbol_names = self.load_symbol_names()
        with open(self.input_header_file(), "r") as f:
            lines = f.readlines()
        with open(self.output_header_file(), "w") as f:
            aliased_symbol = None
            aliases = []
            symbol_declaration = []

            # Ignore old alias declarations for idempotency
            ignored_symbol = None
            prev_aliases = []

            for line, in_comment, _in_docstring, in_directive, is_deprecated_macro in self._input_header_lines(lines):
                if is_deprecated_macro:
                    # Skip existing deprecated macros (we ensure that this line don't contain symbol names)
                    continue

                if not in_comment and not in_directive and aliased_symbol is None and ignored_symbol is None:
                    match = self.slist.NAME_REGEX.search(line)
                    if match:
                        symbol: str = match[1]
                        if symbol in symbol_names:  # safety check
                            if symbol in prev_aliases:
                                # This symbol is an alias that already has an
                                # autogenerated declaration
                                ignored_symbol = symbol
                            else:
                                # Starting a new non-alias symbol
                                prev_aliases = []
                                aliases = self.symbol_aliases.get(symbol, [])
                                if aliases:
                                    aliased_symbol = symbol

                if ignored_symbol is not None:
                    if ";" in line.split("//", 1)[0]:
                        # Symbol declaration finished; stop ignoring
                        ignored_symbol = None
                    continue

                f.write(line)
                if aliased_symbol is not None:
                    symbol_declaration.append(line)
                    if ";" in line.split("//", 1)[0]:
                        # Symbol declaration is finished; add aliases
                        for alias in aliases:
                            if mark_as_deprecated:
                                f.write(f'DEPRECATED("{aliased_symbol}")\n')

                            f.write(
                                symbol_declaration[0].replace(aliased_symbol, alias, 1)
                            )
                            f.write("".join(symbol_declaration[1:]))
                            add_count += 1
                        aliased_symbol = None
                        prev_aliases = aliases
                        aliases = []
                        symbol_declaration = []
        self.commit_header_file()
        return add_count
    
    def add_deprecated_macro(self):
        """
        Adds the `DEPRECATED(name)` macro to the header file if it doesn't exist yet.
        This is a workaround for compatibility with clang-format
        (see https://stackoverflow.com/questions/76898417/make-clang-format-break-after-attribute)
        """

        macro = textwrap.dedent("""
            #ifndef DEPRECATED
            #define DEPRECATED(name) __attribute__((deprecated("Renamed to '" name "'")))
            #endif
        """)

        with open(self.input_header_file(), "r") as f:
            lines = f.readlines()

        with open(self.output_header_file(), "w") as f:
            in_guard = False
            for line in lines:
                if line.strip() == "#ifndef DEPRECATED":
                    in_guard = True

                if not in_guard:
                    f.write(line)
                
                if in_guard and line.strip() == "#endif":
                    in_guard = False

                if HeaderAugmenter.INCLUDE_GUARD_RE.match(line):
                    f.write(macro)

        self.commit_header_file()

    def get_docstring(self, symbol: str) -> Optional[str]:
        if symbol not in self.symbol_descriptions:
            return None
        return self.formatter.format_docstring(self.symbol_descriptions[symbol])

    def add_docstrings(self) -> int:
        add_count = 0
        symbol_names = self.load_symbol_names()
        buffered_attribute = None
        with open(self.input_header_file(), "r") as f:
            lines = f.readlines()
        with open(self.output_header_file(), "w") as f:
            for line, in_comment, in_docstring, in_directive, is_deprecated_macro in self._input_header_lines(lines):
                if is_deprecated_macro:
                    # Write attributes after docstrings
                    buffered_attribute = line
                    continue
                if in_docstring:
                    # Skip over docstrings
                    continue
                if not in_comment and not in_directive:
                    match = self.slist.NAME_REGEX.search(line)
                    if match:
                        symbol = match[1]
                        if symbol in symbol_names:  # safety check
                            # Insert a docstring before writing this line
                            # NOTE: This assumes the symbol name is on the
                            # first line of the symbol declaration
                            docstring = self.get_docstring(symbol)
                            if docstring is not None:
                                f.write(HeaderAugmenter.DOCSTRING_PREAMBLE_LINE)
                                f.write(docstring)

                                if buffered_attribute:
                                    f.write(buffered_attribute)
                                    buffered_attribute = None

                                add_count += 1

                f.write(line)
        self.commit_header_file()
        return add_count


def add_header_content(
    symbol_list_cls: HeaderSymbolList,
    extension: str,
    *,
    aliases: bool = True,
    mark_aliases_as_deprecated: bool = True,
    docstrings: bool = True,
    formatter: Optional[Formatter] = None,
    filter: Optional[str] = None,
    verbosity: int = 0,
):
    for header_file in symbol_list_cls.headers():
        if filter is not None and not fnmatch.fnmatch(header_file, filter):
            continue
        try:
            augmenter = HeaderAugmenter(
                symbol_list_cls(header_file), extension=extension, formatter=formatter
            )
        except ValueError:
            # File doesn't correspond to a symbol file; skip
            continue

        if aliases:
            count = augmenter.add_aliases(mark_aliases_as_deprecated)
            if verbosity >= 2 or (verbosity >= 1 and count > 0):
                print(f"Added {count} alias declaration(s) to {header_file}")

            if count > 0 and mark_aliases_as_deprecated:
                augmenter.add_deprecated_macro()
                if verbosity >= 1:
                    print(f"Added `DEPRECATED` macro to {header_file}")
        if docstrings:
            count = augmenter.add_docstrings()
            if verbosity >= 2 or (verbosity >= 1 and count > 0):
                print(f"Added {count} docstring(s) to {header_file}")


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Add autogenerated content to the C headers using the symbol tables"
    )
    parser.add_argument(
        "-a",
        "--aliases",
        action="store_true",
        help="add alias declarations",
    )
    parser.add_argument(
        "--deprecate-aliases",
        action="store_true",
        help="mark aliases with __attribute__((deprecated)) if --aliases is set"
    )
    parser.add_argument(
        "-d",
        "--docstrings",
        action="store_true",
        help="add Javadoc-style docstrings",
    )
    parser.add_argument(
        "-e",
        "--extension",
        default="",
        help="file extension for the augmented output files; will modify in-place if omitted or empty",
    )
    parser.add_argument(
        "-f",
        "--filter",
        help="Unix filename path filter for header files to process",
    )
    parser.add_argument(
        "-v", "--verbose", action="count", default=0, help="verbosity level"
    )
    args = parser.parse_args()

    # Prefer formatting with clang-format if possible
    try:
        formatter = ClangFormatFormatter()
        if args.verbose:
            print("using clang-format for formatting")
    except Exception:
        formatter = TextWrapFormatter()
        if args.verbose:
            print(
                "clang-format not found; "
                + f"using textwrap for formatting (linewidth={formatter.linewidth})"
            )

    add_header_content(
        FunctionList,
        args.extension,
        aliases=args.aliases,
        mark_aliases_as_deprecated=args.deprecate_aliases,
        docstrings=args.docstrings,
        formatter=formatter,
        filter=args.filter,
        verbosity=args.verbose,
    )
    add_header_content(
        DataList,
        args.extension,
        aliases=args.aliases,
        mark_aliases_as_deprecated=args.deprecate_aliases,
        docstrings=args.docstrings,
        formatter=formatter,
        filter=args.filter,
        verbosity=args.verbose,
    )