diff options
Diffstat (limited to 'pw_docgen')
| -rw-r--r-- | pw_docgen/docs.gni | 166 | ||||
| -rw-r--r-- | pw_docgen/docs.rst | 127 | ||||
| -rw-r--r-- | pw_docgen/py/BUILD.gn | 4 | ||||
| -rw-r--r-- | pw_docgen/py/pw_docgen/docgen.py | 32 | ||||
| -rw-r--r-- | pw_docgen/py/pw_docgen/sphinx/kconfig.py | 137 | ||||
| -rw-r--r-- | pw_docgen/py/pw_docgen/sphinx/module_metadata.py | 329 | ||||
| -rw-r--r-- | pw_docgen/py/pw_docgen/sphinx/pigweed_live.py | 130 | ||||
| -rw-r--r-- | pw_docgen/py/pw_docgen/sphinx/seed_metadata.py | 133 | ||||
| -rw-r--r-- | pw_docgen/py/setup.cfg | 1 | ||||
| -rw-r--r-- | pw_docgen/py/setup.py | 18 |
10 files changed, 824 insertions, 253 deletions
diff --git a/pw_docgen/docs.gni b/pw_docgen/docs.gni index 2d0ae00cf..ed157f914 100644 --- a/pw_docgen/docs.gni +++ b/pw_docgen/docs.gni @@ -22,7 +22,10 @@ declare_args() { pw_docgen_BUILD_DOCS = false # Set to enable Google Analytics tracking of generated docs. - pw_docs_google_analytics_id = "" + pw_docgen_GOOGLE_ANALYTICS_ID = "" + + # Set to define the number of parallel threads to use during the Sphinx build. + pw_docgen_THREADS = "" } # Defines a group of documentation files and assets. @@ -34,40 +37,38 @@ declare_args() { # report_deps: Report card targets on which documentation depends. # other_deps: Dependencies on any other types of targets. template("pw_doc_group") { - if (defined(invoker.sources)) { - _sources = invoker.sources - } else { - _sources = [] - } - - if (defined(invoker.inputs)) { - _inputs = invoker.inputs - } else { - _inputs = [] - } - assert(defined(invoker.sources) || defined(invoker.inputs), "pw_doc_group requires at least one of sources or inputs") - _all_deps = [] - if (defined(invoker.group_deps)) { - _all_deps += invoker.group_deps - } - if (defined(invoker.report_deps)) { - _all_deps += invoker.report_deps - } - if (defined(invoker.other_deps)) { - _all_deps += invoker.other_deps - } - # Create a group containing the source and input files so that docs are # rebuilt on file modifications. pw_input_group(target_name) { + _sources = [] + if (defined(invoker.sources)) { + _sources = invoker.sources + } + + _inputs = [] + if (defined(invoker.inputs)) { + _inputs = invoker.inputs + } + metadata = { pw_doc_sources = rebase_path(_sources, root_build_dir) pw_doc_inputs = rebase_path(_inputs, root_build_dir) } - deps = _all_deps + + deps = [] + if (defined(invoker.group_deps)) { + deps += invoker.group_deps + } + if (defined(invoker.report_deps)) { + deps += invoker.report_deps + } + if (defined(invoker.other_deps)) { + deps += invoker.other_deps + } + inputs = _sources + _inputs } } @@ -76,6 +77,10 @@ template("pw_doc_group") { # # Args: # deps: List of pw_doc_group targets. +# python_metadata_deps: Python-related dependencies that are only used as deps +# for generating Python package metadata list, not the +# overall documentation generation. This should rarely +# be used by non-Pigweed code. # sources: Top-level documentation .rst source files. # conf: Configuration script (conf.py) for Sphinx. # output_directory: Path to directory to which HTML output is rendered. @@ -89,61 +94,68 @@ template("pw_doc_gen") { assert(defined(invoker.output_directory), "pw_doc_gen requires an 'output_directory' argument") - # Collects all dependency metadata into a single JSON file. - _metadata_file_target = "${target_name}_metadata" - generated_file(_metadata_file_target) { - outputs = [ "$target_gen_dir/$target_name.json" ] - data_keys = [ - "pw_doc_sources", - "pw_doc_inputs", - ] - output_conversion = "json" - deps = invoker.deps - } - - _script_args = [ - "--gn-root", - rebase_path("//", root_build_dir), - "--gn-gen-root", - rebase_path(root_gen_dir, root_build_dir) + "/", - "--sphinx-build-dir", - rebase_path("$target_gen_dir/pw_docgen_tree", root_build_dir), - "--conf", - rebase_path(invoker.conf, root_build_dir), - "--out-dir", - rebase_path(invoker.output_directory, root_build_dir), - ] - - # Enable Google Analytics if a measurement ID is provided - if (pw_docs_google_analytics_id != "") { - _script_args += [ - "--google-analytics-id", - pw_docs_google_analytics_id, - ] - } - - # Metadata JSON file path. - _script_args += [ "--metadata" ] - _script_args += - rebase_path(get_target_outputs(":$_metadata_file_target"), root_build_dir) - - _script_args += rebase_path(invoker.sources, root_build_dir) - - # Required to set the PYTHONPATH for any automodule/class/function RST - # directives. - _python_metadata_deps = [ "$dir_pw_docgen/py" ] - if (defined(invoker.python_metadata_deps)) { - _python_metadata_deps += invoker.python_metadata_deps - } - if (pw_docgen_BUILD_DOCS) { + # Collects all dependency metadata into a single JSON file. + _metadata_file_target = "${target_name}_metadata" + generated_file(_metadata_file_target) { + outputs = [ "$target_gen_dir/$target_name.json" ] + data_keys = [ + "pw_doc_sources", + "pw_doc_inputs", + ] + output_conversion = "json" + deps = invoker.deps + } + pw_python_action(target_name) { - script = "$dir_pw_docgen/py/pw_docgen/docgen.py" - args = _script_args + module = "pw_docgen.docgen" + args = [ + "--gn-root", + rebase_path("//", root_build_dir), + "--gn-gen-root", + rebase_path(root_gen_dir, root_build_dir) + "/", + "--sphinx-build-dir", + rebase_path("$target_gen_dir/pw_docgen_tree", root_build_dir), + "--conf", + rebase_path(invoker.conf, root_build_dir), + "--out-dir", + rebase_path(invoker.output_directory, root_build_dir), + ] + + # Enable Google Analytics if a measurement ID is provided + if (pw_docgen_GOOGLE_ANALYTICS_ID != "") { + args += [ + "--google-analytics-id", + pw_docgen_GOOGLE_ANALYTICS_ID, + ] + } + + # Override the default number of threads for the Sphinx build. + if (pw_docgen_THREADS != "") { + args += [ + "-j", + pw_docgen_THREADS, + ] + } + + # Metadata JSON file path. + args += [ "--metadata" ] + + rebase_path(get_target_outputs(":$_metadata_file_target"), + root_build_dir) + + args += rebase_path(invoker.sources, root_build_dir) + + python_deps = [ "$dir_pw_docgen/py" ] deps = [ ":$_metadata_file_target" ] - python_metadata_deps = _python_metadata_deps - inputs = [ invoker.conf ] - inputs += invoker.sources + + # Required to set the PYTHONPATH for any automodule/class/function RST + # directives. + python_metadata_deps = [ "$dir_pw_docgen/py" ] + if (defined(invoker.python_metadata_deps)) { + python_metadata_deps += invoker.python_metadata_deps + } + + inputs = [ invoker.conf ] + invoker.sources stamp = true } } else { diff --git a/pw_docgen/docs.rst b/pw_docgen/docs.rst index 6971384a3..b36de7524 100644 --- a/pw_docgen/docs.rst +++ b/pw_docgen/docs.rst @@ -71,17 +71,20 @@ groups, causing them to be built with it. * ``group_deps``: Other ``pw_doc_group`` targets required by this one. * ``report_deps``: Report card generating targets (e.g. ``pw_size_diff``) on which the docs depend. +* ``other_deps``: Any other GN targets that should be run before this + ``pw_doc_group`` runs that is not included in one of the above ``dep`` + categories. **Example** -.. code:: +.. code-block:: - pw_doc_group("my_doc_group") { - sources = [ "docs.rst" ] - inputs = [ "face-with-tears-of-joy-emoji.svg" ] - group_deps = [ ":sub_doc_group" ] - report_deps = [ ":my_size_report" ] - } + pw_doc_group("my_doc_group") { + sources = [ "docs.rst" ] + inputs = [ "face-with-tears-of-joy-emoji.svg" ] + group_deps = [ ":sub_doc_group" ] + report_deps = [ ":my_size_report" ] + } pw_doc_gen __________ @@ -100,19 +103,22 @@ to tie everything together. * ``index``: Path to the top-level ``index.rst`` file. * ``output_directory``: Directory in which to render HTML output. * ``deps``: List of all ``pw_doc_group`` targets required for the documentation. +* ``python_metadata_deps``: Python-related dependencies that are only used as + deps for generating Python package metadata list, not the overall + documentation generation. This should rarely be used by non-Pigweed code. **Example** -.. code:: +.. code-block:: - pw_doc_gen("my_docs") { - conf = "//my_docs/conf.py" - index = "//my_docs/index.rst" - output_directory = target_gen_dir - deps = [ - "//my_module:my_doc_group", - ] - } + pw_doc_gen("my_docs") { + conf = "//my_docs/conf.py" + index = "//my_docs/index.rst" + output_directory = target_gen_dir + deps = [ + "//my_module:my_doc_group", + ] + } Generating Documentation ------------------------ @@ -123,18 +129,18 @@ using a subset of Pigweed's core documentation. Consider the following target in ``$dir_pigweed/docs/BUILD.gn``: -.. code:: +.. code-block:: - pw_doc_gen("docs") { - conf = "conf.py" - index = "index.rst" - output_directory = target_gen_dir - deps = [ - "$dir_pw_bloat:docs", - "$dir_pw_docgen:docs", - "$dir_pw_preprocessor:docs", - ] - } + pw_doc_gen("docs") { + conf = "conf.py" + index = "index.rst" + output_directory = target_gen_dir + deps = [ + "$dir_pw_bloat:docs", + "$dir_pw_docgen:docs", + "$dir_pw_preprocessor:docs", + ] + } A documentation tree is created under the output directory. Each of the sources and inputs in the target's dependency graph is copied under this tree in the @@ -142,19 +148,19 @@ same directory structure as they appear under the root GN build directory (``$dir_pigweed`` in this case). The ``conf.py`` and ``index.rst`` provided directly to the ``pw_doc_gen`` template are copied in at the root of the tree. -.. code:: +.. code-block:: - out/gen/docs/pw_docgen_tree/ - ├── conf.py - ├── index.rst - ├── pw_bloat - │ ├── bloat.rst - │ └── examples - │ └── simple_bloat.rst - ├── pw_docgen - │ └── docgen.rst - └── pw_preprocessor - └── docs.rst + out/gen/docs/pw_docgen_tree/ + ├── conf.py + ├── index.rst + ├── pw_bloat + │ ├── bloat.rst + │ └── examples + │ └── simple_bloat.rst + ├── pw_docgen + │ └── docgen.rst + └── pw_preprocessor + └── docs.rst This is the documentation tree which gets passed to Sphinx to build HTML output. Imports within documentation files must be relative to this structure. In @@ -190,12 +196,8 @@ example: :name: pw_string :tagline: Efficient, easy, and safe string manipulation :status: stable - :languages: C++14, C++17 + :languages: C++17, Rust :code-size-impact: 500 to 1500 bytes - :get-started: module-pw_string-get-started - :design: module-pw_string-design - :guides: module-pw_string-guide - :api: module-pw_string-api Module sales pitch goes here! @@ -205,14 +207,9 @@ _________________ - ``tagline``: A very short tagline that summarizes the module (required) - ``status``: One of ``experimental``, ``unstable``, and ``stable`` (required) - ``is-deprecated``: A flag indicating that the module is deprecated -- ``languages``: A comma-separated list of languages the module supports +- ``languages``: A comma-separated list of languages the module supports. If + the language has API docs (Rust), they will be linked from the metadata block. - ``code-size-impact``: A summarize of the average code size impact -- ``get-started``: A reference to the getting started section (required) -- ``tutorials``: A reference to the tutorials section -- ``guides``: A reference to the guides section -- ``design``: A reference to the design considerations section (required) -- ``concepts``: A reference to the concepts documentation -- ``api``: A reference to the API documentation google_analytics ---------------- @@ -225,3 +222,31 @@ automatically based on the value of the GN argument ``pw_docs_google_analytics_id``, allowing you to control whether tracking is enabled or not in your build configuration. Typically, you would only enable this for documentation builds intended for deployment on the web. + +Debugging Pigweed's Sphinx extensions +------------------------------------- +To step through your Pigweed extension code with +`pdb <https://docs.python.org/3/library/pdb.html>`_: + +#. Set a breakpoint in your extension code: + + .. code-block:: + + breakpoint() + +#. Build ``python.install`` to install the code change into the bootstrap venv + (``environment/pigweed-venv/lib/python3.8/site-packages/pw_docgen``): + + .. code-block:: + + ninja -C out python.install + +#. Manually invoke Sphinx to build the docs and trigger your breakpoint: + + .. code-block:: + + cd out + sphinx-build -W -b html -d docs/gen/docs/help docs/gen/docs/pw_docgen_tree docs/gen/docs/html -v -v -v + + You should see build output from Sphinx. The build should pause at your + breakpoint and you should then see pdb's prompt (``(Pdb)``). diff --git a/pw_docgen/py/BUILD.gn b/pw_docgen/py/BUILD.gn index 55c48aaeb..a52f3ab31 100644 --- a/pw_docgen/py/BUILD.gn +++ b/pw_docgen/py/BUILD.gn @@ -20,14 +20,16 @@ pw_python_package("py") { setup = [ "pyproject.toml", "setup.cfg", - "setup.py", ] sources = [ "pw_docgen/__init__.py", "pw_docgen/docgen.py", "pw_docgen/sphinx/__init__.py", "pw_docgen/sphinx/google_analytics.py", + "pw_docgen/sphinx/kconfig.py", "pw_docgen/sphinx/module_metadata.py", + "pw_docgen/sphinx/pigweed_live.py", + "pw_docgen/sphinx/seed_metadata.py", ] pylintrc = "$dir_pigweed/.pylintrc" mypy_ini = "$dir_pigweed/.mypy.ini" diff --git a/pw_docgen/py/pw_docgen/docgen.py b/pw_docgen/py/pw_docgen/docgen.py index 764717d4a..24040e7cf 100644 --- a/pw_docgen/py/pw_docgen/docgen.py +++ b/pw_docgen/py/pw_docgen/docgen.py @@ -50,6 +50,13 @@ def parse_args() -> argparse.Namespace: '--conf', required=True, help='Path to conf.py file for Sphinx' ) parser.add_argument( + '-j', + '--parallel', + type=int, + default=os.cpu_count(), + help='Number of parallel processes to run', + ) + parser.add_argument( '--gn-root', required=True, help='Root of the GN build tree' ) parser.add_argument( @@ -78,13 +85,25 @@ def parse_args() -> argparse.Namespace: def build_docs( - src_dir: str, dst_dir: str, google_analytics_id: Optional[str] = None + src_dir: str, + dst_dir: str, + parallel: int, + google_analytics_id: Optional[str] = None, ) -> int: """Runs Sphinx to render HTML documentation from a doc tree.""" # TODO(frolv): Specify the Sphinx script from a prebuilts path instead of # requiring it in the tree. - command = ['sphinx-build', '-W', '-b', 'html', '-d', f'{dst_dir}/help'] + command = [ + 'sphinx-build', + '-W', + '-j', + str(parallel), + '-b', + 'html', + '-d', + f'{dst_dir}/help', + ] if google_analytics_id is not None: command.append(f'-Dgoogle_analytics_id={google_analytics_id}') @@ -138,8 +157,8 @@ def main() -> int: if os.path.exists(args.sphinx_build_dir): shutil.rmtree(args.sphinx_build_dir) - # TODO(b/235349854): Printing the header causes unicode problems on Windows. - # Disabled for now; re-enable once the root issue is fixed. + # TODO: b/235349854 - Printing the header causes unicode problems on + # Windows. Disabled for now; re-enable once the root issue is fixed. # print(SCRIPT_HEADER) copy_doc_tree(args) @@ -147,7 +166,10 @@ def main() -> int: print('-' * 80, flush=True) return build_docs( - args.sphinx_build_dir, args.out_dir, args.google_analytics_id + args.sphinx_build_dir, + args.out_dir, + args.parallel, + args.google_analytics_id, ) diff --git a/pw_docgen/py/pw_docgen/sphinx/kconfig.py b/pw_docgen/py/pw_docgen/sphinx/kconfig.py new file mode 100644 index 000000000..c18f3d816 --- /dev/null +++ b/pw_docgen/py/pw_docgen/sphinx/kconfig.py @@ -0,0 +1,137 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Auto-generate the Kconfig reference in //docs/os/zephyr/kconfig.rst""" + + +import os +import re +from typing import Iterable, Dict + +import docutils +from docutils.core import publish_doctree +from sphinx.application import Sphinx +from sphinx.addnodes import document + + +try: + import kconfiglib # type: ignore + + KCONFIGLIB_AVAILABLE = True +except ImportError: + KCONFIGLIB_AVAILABLE = False + + +def rst_to_doctree(rst: str) -> Iterable[docutils.nodes.Node]: + """Convert raw reStructuredText into doctree nodes.""" + # TODO: b/288127315 - Properly resolve references within the rst so that + # links are generated more robustly. + while ':ref:`module-' in rst: + rst = re.sub( + r':ref:`module-(.*?)`', r'`\1 <https://pigweed.dev/\1>`_', rst + ) + doctree = publish_doctree(rst) + return doctree.children + + +def create_source_paragraph(name_and_loc: str) -> Iterable[docutils.nodes.Node]: + """Convert kconfiglib's name and location string into a source code link.""" + start = name_and_loc.index('pw_') + end = name_and_loc.index(':') + file_path = name_and_loc[start:end] + url = f'https://cs.opensource.google/pigweed/pigweed/+/main:{file_path}' + link = f'`//{file_path} <{url}>`_' + return rst_to_doctree(f'Source: {link}') + + +def process_node( + node: kconfiglib.MenuNode, parent: docutils.nodes.Node +) -> None: + """Recursively generate documentation for the Kconfig nodes.""" + while node: + if node.item == kconfiglib.MENU: + name = node.prompt[0] + # All auto-generated sections must have an ID or else the + # get_secnumber() function in Sphinx's HTML5 writer throws an + # IndexError. + menu_section = docutils.nodes.section(ids=[name]) + menu_section += docutils.nodes.title(text=f'{name} options') + if node.list: + process_node(node.list, menu_section) + parent += menu_section + elif isinstance(node.item, kconfiglib.Symbol): + name = f'CONFIG_{node.item.name}' + symbol_section = docutils.nodes.section(ids=[name]) + symbol_section += docutils.nodes.title(text=name) + symbol_section += docutils.nodes.paragraph( + text=f'Type: {kconfiglib.TYPE_TO_STR[node.item.type]}' + ) + if node.item.defaults: + try: + default_value = node.item.defaults[0][0].str_value + symbol_section += docutils.nodes.paragraph( + text=f'Default value: {default_value}' + ) + # If the data wasn't found, just contine trying to process + # rest of the documentation for the node. + except IndexError: + pass + if node.item.ranges: + try: + low = node.item.ranges[0][0].str_value + high = node.item.ranges[0][1].str_value + symbol_section += docutils.nodes.paragraph( + text=f'Range of valid values: {low} to {high}' + ) + except IndexError: + pass + if node.prompt: + try: + symbol_section += docutils.nodes.paragraph( + text=f'Description: {node.prompt[0]}' + ) + except IndexError: + pass + if node.help: + symbol_section += rst_to_doctree(node.help) + if node.list: + process_node(node.list, symbol_section) + symbol_section += create_source_paragraph(node.item.name_and_loc) + parent += symbol_section + # TODO: b/288127315 - Render choices? + # elif isinstance(node.item, kconfiglib.Choice): + node = node.next + + +def generate_kconfig_reference(_, doctree: document, docname: str) -> None: + """Parse the Kconfig and kick off the doc generation process.""" + if 'docs/os/zephyr/kconfig' not in docname: + return + # Assume that the new content should be appended to the last section + # in the doctree. + for child in doctree.children: + if isinstance(child, docutils.nodes.section): + root = child + pw_root = os.environ['PW_ROOT'] + file_path = f'{pw_root}/Kconfig.zephyr' + kconfig = kconfiglib.Kconfig(file_path) + # There's no need to render kconfig.top_node (the main menu) or + # kconfig.top_node.list (ZEPHYR_PIGWEED_MODULE). + process_node(kconfig.top_node.list.next, root) + + +def setup(app: Sphinx) -> Dict[str, bool]: + """Initialize the Sphinx extension.""" + if KCONFIGLIB_AVAILABLE: + app.connect('doctree-resolved', generate_kconfig_reference) + return {'parallel_read_safe': True, 'parallel_write_safe': True} diff --git a/pw_docgen/py/pw_docgen/sphinx/module_metadata.py b/pw_docgen/py/pw_docgen/sphinx/module_metadata.py index d6fe05497..fad1736d7 100644 --- a/pw_docgen/py/pw_docgen/sphinx/module_metadata.py +++ b/pw_docgen/py/pw_docgen/sphinx/module_metadata.py @@ -13,18 +13,76 @@ # the License. """Sphinx directives for Pigweed module metadata""" -from typing import List +from dataclasses import dataclass +from typing import cast, Dict, List, Optional, TypeVar, Union + +# We use BeautifulSoup for certain docs rendering features. It may not be +# available in downstream projects. If so, no problem. We fall back to simpler +# docs rendering. +# pylint: disable=import-error +try: + from bs4 import BeautifulSoup # type: ignore + from bs4.element import Tag as HTMLTag # type: ignore + + bs_enabled = True +except ModuleNotFoundError: + bs_enabled = False +# pylint: enable=import-error import docutils +from docutils import nodes +import docutils.statemachine # pylint: disable=consider-using-from-import import docutils.parsers.rst.directives as directives # type: ignore # pylint: enable=consider-using-from-import from sphinx.application import Sphinx as SphinxApplication +from sphinx.environment import BuildEnvironment from sphinx.util.docutils import SphinxDirective -from sphinx_design.badges_buttons import ButtonRefDirective # type: ignore -from sphinx_design.cards import CardDirective # type: ignore + +from sphinx_design.cards import CardDirective + +EnvAttrT = TypeVar('EnvAttrT') + + +@dataclass +class ParsedBody: + topnav: str + body_without_topnav: str + + +class EnvMetadata: + """Easier access to the Sphinx `env` for custom metadata. + + You can store things in the Sphinx `env`, which is just a dict. But each + time you do, you have to handle the possibility that the key you want + hasn't been set yet, and set it to a default. The `env` is also untyped, + so you have to cast the value you get to whatever type you expect it to be. + + Or you can use this class to define your metadata keys up front, and just + access them like: `value = EnvMetadata(env).my_value` + + ... which will handle initializing the value if it hasn't been yet and + provide you a typed result. + """ + + def __init__(self, env: BuildEnvironment): + self._env = env + + def _get_env_attr(self, attr: str, default: EnvAttrT) -> EnvAttrT: + if not hasattr(self._env, attr): + value: EnvAttrT = default + setattr(self._env, attr, value) + else: + value = getattr(self._env, attr) + + return value + + @property + def pw_parsed_bodies(self) -> Dict[str, ParsedBody]: + default: Dict[str, ParsedBody] = {} + return self._get_env_attr('pw_module_nav', default) def status_choice(arg): @@ -32,15 +90,19 @@ def status_choice(arg): def status_badge(module_status: str) -> str: + """Given a module status, return the status badge for rendering.""" role = ':bdg-primary:' return role + f'`{module_status.title()}`' -def cs_url(module_name: str): +def cs_url(module_name: str) -> str: + """Return the codesearch URL for the given module.""" return f'https://cs.opensource.google/pigweed/pigweed/+/main:{module_name}/' -def concat_tags(*tag_lists: List[str]): +def concat_tags(*tag_lists: List[str]) -> List[str]: + """Given a list of tag lists, return them concat'ed and ready for render.""" + all_tags = tag_lists[0] for tag_list in tag_lists[1:]: @@ -51,6 +113,27 @@ def concat_tags(*tag_lists: List[str]): return all_tags +def create_topnav( + subtitle: str, + extra_classes: Optional[List[str]] = None, +) -> nodes.Node: + """Create the nodes for the top title and navigation bar.""" + + topnav_classes = ( + ['pw-topnav'] + extra_classes if extra_classes is not None else [] + ) + + topnav_container = nodes.container(classes=topnav_classes) + + subtitle_node = nodes.paragraph( + classes=['pw-topnav-subtitle'], + text=subtitle, + ) + + topnav_container += subtitle_node + return topnav_container + + class PigweedModuleDirective(SphinxDirective): """Directive registering module metadata, rendering title & info card.""" @@ -65,74 +148,27 @@ class PigweedModuleDirective(SphinxDirective): 'languages': directives.unchanged, 'code-size-impact': directives.unchanged, 'facade': directives.unchanged, - 'get-started': directives.unchanged_required, - 'tutorials': directives.unchanged, - 'guides': directives.unchanged, - 'concepts': directives.unchanged, - 'design': directives.unchanged_required, - 'api': directives.unchanged, + 'nav': directives.unchanged_required, } - def try_get_option(self, option: str): - try: - return self.options[option] - except KeyError: - raise self.error(f' :{option}: option is required') + def _try_get_option(self, option: str): + """Try to get an option by name and raise on failure.""" - def maybe_get_option(self, option: str): try: return self.options[option] except KeyError: - return None - - def create_section_button(self, title: str, ref: str): - node = docutils.nodes.list_item(classes=['pw-module-section-button']) - node += ButtonRefDirective( - name='', - arguments=[ref], - options={'color': 'primary'}, - content=[title], - lineno=0, - content_offset=0, - block_text='', - state=self.state, - state_machine=self.state_machine, - ).run() - - return node - - def register_metadata(self): - module_name = self.try_get_option('name') - - if 'facade' in self.options: - facade = self.options['facade'] - - # Initialize the module relationship dict if needed - if not hasattr(self.env, 'pw_module_relationships'): - self.env.pw_module_relationships = {} - - # Initialize the backend list for this facade if needed - if facade not in self.env.pw_module_relationships: - self.env.pw_module_relationships[facade] = [] - - # Add this module as a backend of the provided facade - self.env.pw_module_relationships[facade].append(module_name) - - if 'is-deprecated' in self.options: - # Initialize the deprecated modules list if needed - if not hasattr(self.env, 'pw_modules_deprecated'): - self.env.pw_modules_deprecated = [] + raise self.error(f' :{option}: option is required') - self.env.pw_modules_deprecated.append(module_name) + def _maybe_get_option(self, option: str): + """Try to get an option by name and return None on failure.""" + return self.options.get(option, None) - def run(self): - tagline = docutils.nodes.paragraph( - text=self.try_get_option('tagline'), - classes=['section-subtitle'], - ) + def run(self) -> List[nodes.Node]: + module_name = self._try_get_option('name') + tagline = self._try_get_option('tagline') status_tags: List[str] = [ - status_badge(self.try_get_option('status')), + status_badge(self._try_get_option('status')), ] if 'is-deprecated' in self.options: @@ -145,17 +181,25 @@ class PigweedModuleDirective(SphinxDirective): if len(languages) > 0: for language in languages: - language_tags.append(f':bdg-info:`{language.strip()}`') + language = language.strip() + if language == 'Rust': + language_tags.append( + f':bdg-link-info:`{language}' + + f'</rustdoc/{module_name}>`' + ) + else: + language_tags.append(f':bdg-info:`{language}`') code_size_impact = [] - if code_size_text := self.maybe_get_option('code-size-impact'): + if code_size_text := self._maybe_get_option('code-size-impact'): code_size_impact.append(f'**Code Size Impact:** {code_size_text}') # Move the directive content into a section that we can render wherever # we want. - content = docutils.nodes.paragraph() - self.state.nested_parse(self.content, 0, content) + raw_content = cast(List[str], self.content) # type: ignore + content = nodes.paragraph() + self.state.nested_parse(raw_content, 0, content) # The card inherits its content from this node's content, which we've # already pulled out. So we can replace this node's content with the @@ -170,55 +214,140 @@ class PigweedModuleDirective(SphinxDirective): options={}, ) - # Create the top-level section buttons. - section_buttons = docutils.nodes.bullet_list( - classes=['pw-module-section-buttons'] + topbar = create_topnav( + tagline, + ['pw-module-index'], ) - # This is the pattern for required sections. - section_buttons += self.create_section_button( - 'Get Started', self.try_get_option('get-started') - ) + return [topbar, card, content] - # This is the pattern for optional sections. - if (tutorials_ref := self.maybe_get_option('tutorials')) is not None: - section_buttons += self.create_section_button( - 'Tutorials', tutorials_ref - ) - if (guides_ref := self.maybe_get_option('guides')) is not None: - section_buttons += self.create_section_button('Guides', guides_ref) +class PigweedModuleSubpageDirective(PigweedModuleDirective): + """Directive registering module metadata, rendering title & info card.""" - if (concepts_ref := self.maybe_get_option('concepts')) is not None: - section_buttons += self.create_section_button( - 'Concepts', concepts_ref - ) + required_arguments = 0 + final_argument_whitespace = True + has_content = True + option_spec = { + 'name': directives.unchanged_required, + 'tagline': directives.unchanged_required, + 'nav': directives.unchanged_required, + } + + def run(self) -> List[nodes.Node]: + tagline = self._try_get_option('tagline') - section_buttons += self.create_section_button( - 'Design', self.try_get_option('design') + topbar = create_topnav( + tagline, + ['pw-module-subpage'], ) - if (api_ref := self.maybe_get_option('api')) is not None: - section_buttons += self.create_section_button( - 'API Reference', api_ref - ) + return [topbar] + + +def _parse_body(body: str) -> ParsedBody: + """From the `body` HTML, return the topnav and the body without topnav. + + The fundamental idea is this: Our Sphinx directives can only render nodes + *within* the docutils doc, but we want to elevate the top navbar *outside* + of that doc into the web theme. Sphinx by itself provides no mechanism for + this, since it's model looks something like this: + + ┌──────────────────┐ + │ Theme │ + │ ┌──────────────┐│ When Sphinx builds HTML, the output is plain HTML + │ │ Sphinx HTML ││ with a structure defined by docutils. Themes can + │ │ ││ build *around* that and cascade styles down *into* + │ │ ││ that HTML, but there's no mechanism in the Sphinx + │ └──────────────┘│ build to render docutils nodes in the theme. + └──────────────────┘ + + The escape hatch is this: + - Render things within the Sphinx HTML output (`body`) + - Use Sphinx theme templates to run code during the final render phase + - Extract the HTML from the `body` and insert it in the theme via templates + + So this function extracts the things that we rendered in the `body` but + actually want in the theme (the top navbar), returns them for rendering in + the template, and returns the `body` with those things removed. + """ + if not bs_enabled: + return ParsedBody('', body) + + def _add_class_to_tag(tag: HTMLTag, classname: str) -> None: + tag['class'] = tag.get('class', []) + [classname] # type: ignore + + def _add_classes_to_tag( + tag: HTMLTag, classnames: Union[str, List[str], None] + ) -> None: + tag['class'] = tag.get('class', []) + classnames # type: ignore + + html = BeautifulSoup(body, features='html.parser') + + # Render the doc unchanged, unless it has the module doc topnav + if (topnav := html.find('div', attrs={'class': 'pw-topnav'})) is None: + return ParsedBody('', body) + + assert isinstance(topnav, HTMLTag) + + # Find the topnav title and subtitle + topnav_title = topnav.find('p', attrs={'class': 'pw-topnav-title'}) + topnav_subtitle = topnav.find('p', attrs={'class': 'pw-topnav-subtitle'}) + assert isinstance(topnav_title, HTMLTag) + assert isinstance(topnav_subtitle, HTMLTag) + + # Find the single `h1` element, the doc's canonical title + doc_title = html.find('h1') + assert isinstance(doc_title, HTMLTag) + + topnav_str = '' + + if 'pw-module-index' in topnav['class']: + # Take the standard Sphinx/docutils title and add topnav styling + _add_class_to_tag(doc_title, 'pw-topnav-title') + # Replace the placeholder title in the topnav with the "official" `h1` + topnav_title.replace_with(doc_title) + # Promote the subtitle to `h2` + topnav_subtitle.name = 'h2' + # We're done mutating topnav; write it to string for rendering elsewhere + topnav_str = str(topnav) + # Destroy the instance that was rendered in the document + topnav.decompose() + + elif 'pw-module-subpage' in topnav['class']: + # Take the title from the topnav (the module name), promote it to `h1` + topnav_title.name = 'h1' + # Add the heading link, but pointed to the module index page + heading_link = html.new_tag( + 'a', + attrs={ + 'class': ['headerlink'], + 'href': 'docs.html', + 'title': 'Permalink to module index', + }, + ) + heading_link.string = '#' + topnav_title.append(heading_link) + # Promote the subtitle to `h2` + topnav_subtitle.name = 'h2' + # We're done mutating topnav; write it to string for rendering elsewhere + topnav_str = str(topnav) + # Destroy the instance that was rendered in the document + topnav.decompose() - return [tagline, section_buttons, content, card] + return ParsedBody(topnav_str, str(html)) -def build_backend_lists(app, _doctree, _fromdocname): - env = app.builder.env +def setup_parse_body(_app, _pagename, _templatename, context, _doctree): + def parse_body(body: str) -> ParsedBody: + return _parse_body(body) - if not hasattr(env, 'pw_module_relationships'): - env.pw_module_relationships = {} + context['parse_body'] = parse_body def setup(app: SphinxApplication): app.add_directive('pigweed-module', PigweedModuleDirective) - - # At this event, the documents and metadata have been generated, and now we - # can modify the doctree to reflect the metadata. - app.connect('doctree-resolved', build_backend_lists) + app.add_directive('pigweed-module-subpage', PigweedModuleSubpageDirective) return { 'parallel_read_safe': True, diff --git a/pw_docgen/py/pw_docgen/sphinx/pigweed_live.py b/pw_docgen/py/pw_docgen/sphinx/pigweed_live.py new file mode 100644 index 000000000..d7382cd38 --- /dev/null +++ b/pw_docgen/py/pw_docgen/sphinx/pigweed_live.py @@ -0,0 +1,130 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Docs widget that provides up-to-date info about the next Pigweed Live.""" + + +import datetime +import sys +from typing import Dict, List + +from docutils import nodes +from docutils.parsers.rst import Directive +from sphinx.application import Sphinx + +try: + import pytz # type: ignore + + PYTZ_AVAILABLE = True +except ImportError: + PYTZ_AVAILABLE = False + + +class PigweedLiveDirective(Directive): + """Generates the up-to-date Pigweed Live info.""" + + datetime_format = '%Y-%m-%d %H:%M:%S' + # TODO: b/303859828 - Update this data sometime between 2024-09-23 + # and 2024-10-07. + meetings = [ + '2023-10-09 13:00:00', + '2023-10-23 13:00:00', + '2023-11-06 13:00:00', + # 2023-11-20 skipped since it's a holiday(ish) + '2023-12-04 13:00:00', + '2023-12-18 13:00:00', + # 2024-01-01 and 2024-01-15 are skipped because they're holidays. + '2024-01-29 13:00:00', + '2024-02-12 13:00:00', + '2024-02-26 13:00:00', + '2024-03-11 13:00:00', + '2024-03-25 13:00:00', + '2024-04-08 13:00:00', + '2024-04-22 13:00:00', + '2024-05-06 13:00:00', + '2024-05-20 13:00:00', + '2024-06-03 13:00:00', + '2024-06-17 13:00:00', + '2024-07-01 13:00:00', + '2024-07-15 13:00:00', + '2024-07-29 13:00:00', + '2024-08-12 13:00:00', + '2024-08-26 13:00:00', + '2024-09-09 13:00:00', + '2024-09-23 13:00:00', + '2024-10-07 13:00:00', + ] + timezone = pytz.timezone('US/Pacific') + + def run(self) -> List[nodes.Node]: + return [self._make_paragraph()] + + def _make_paragraph(self) -> nodes.Node: + next_meeting = self._find_next_meeting() + paragraph = nodes.paragraph() + paragraph += nodes.Text('Our next Pigweed Live is ') + meeting_text = nodes.strong() + meeting_text += nodes.Text(next_meeting) + paragraph += meeting_text + paragraph += nodes.Text( + ( + ". Please join us to discuss what's new in Pigweed and " + "anything else Pigweed-related. Or stop in just to say hi and " + "meet the team! You'll find a link for the meeting in the " + "#pigweed-live channel of our " + ) + ) + link = nodes.reference(refuri='https://discord.gg/M9NSeTA') + link += nodes.Text('Discord') + paragraph += link + paragraph += nodes.Text( + ( + '. We meet bi-weekly. The meeting is public. Everyone is ' + 'welcome to join.' + ) + ) + return paragraph + + def _find_next_meeting(self) -> str: + current_datetime = self.timezone.localize(datetime.datetime.now()) + next_meeting = None + for meeting in self.meetings: + unlocalized_datetime = datetime.datetime.strptime( + meeting, self.datetime_format + ) + meeting_datetime = self.timezone.localize(unlocalized_datetime) + if current_datetime > meeting_datetime: + continue + next_meeting = meeting_datetime + break + if next_meeting is None: + sys.exit( + 'ERROR: Pigweed Live meeting data needs to be updated. ' + 'Update the `meetings` list in `PigweedLiveDirective`. ' + 'See b/303859828.' + ) + else: + date = next_meeting.strftime('%a %b %d, %Y') + hour = next_meeting.strftime('%I%p').lstrip('0') + timezone = 'PDT' if next_meeting.dst() else 'PST' + return f'{date} {hour} ({timezone})' + + +def setup(app: Sphinx) -> Dict[str, bool]: + """Initialize the directive.""" + if PYTZ_AVAILABLE: + app.add_directive('pigweed-live', PigweedLiveDirective) + return { + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/pw_docgen/py/pw_docgen/sphinx/seed_metadata.py b/pw_docgen/py/pw_docgen/sphinx/seed_metadata.py new file mode 100644 index 000000000..9134081ea --- /dev/null +++ b/pw_docgen/py/pw_docgen/sphinx/seed_metadata.py @@ -0,0 +1,133 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Sphinx directives for Pigweed SEEDs""" + +from typing import List + +import docutils +from docutils import nodes +import docutils.statemachine + +# pylint: disable=consider-using-from-import +import docutils.parsers.rst.directives as directives # type: ignore + +# pylint: enable=consider-using-from-import +from sphinx.application import Sphinx as SphinxApplication +from sphinx.util.docutils import SphinxDirective + +from sphinx_design.cards import CardDirective + + +def status_choice(arg) -> str: + return directives.choice( + arg, ('open_for_comments', 'last_call', 'accepted', 'rejected') + ) + + +def parse_status(arg) -> str: + """Support variations on the status choices. + + For example, you can use capital letters and spaces. + """ + + return status_choice('_'.join([token.lower() for token in arg.split(' ')])) + + +def status_badge(seed_status: str, badge_status) -> str: + """Given a SEED status, return the status badge for rendering.""" + + return ( + ':bdg-primary:' + if seed_status == badge_status + else ':bdg-secondary-line:' + ) + + +def cl_link(cl_num): + return ( + f'`pwrev/{cl_num} ' + '<https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/' + f'{cl_num}>`_' + ) + + +class PigweedSeedDirective(SphinxDirective): + """Directive registering & rendering SEED metadata.""" + + required_arguments = 0 + final_argument_whitespace = True + has_content = True + option_spec = { + 'number': directives.positive_int, + 'name': directives.unchanged_required, + 'status': parse_status, + 'proposal_date': directives.unchanged_required, + 'cl': directives.positive_int_list, + } + + def _try_get_option(self, option: str): + """Try to get an option by name and raise on failure.""" + + try: + return self.options[option] + except KeyError: + raise self.error(f' :{option}: option is required') + + def run(self) -> List[nodes.Node]: + seed_number = '{:04d}'.format(self._try_get_option('number')) + seed_name = self._try_get_option('name') + status = self._try_get_option('status') + proposal_date = self._try_get_option('proposal_date') + cl_nums = self._try_get_option('cl') + + title = ( + f':fas:`seedling` SEED-{seed_number}: :ref:' + f'`{seed_name}<seed-{seed_number}>`\n' + ) + + self.content = docutils.statemachine.StringList( + [ + ':octicon:`comment-discussion` Status:', + f'{status_badge(status, "open_for_comments")}' + '`Open for Comments`', + ':octicon:`chevron-right`', + f'{status_badge(status, "last_call")}`Last Call`', + ':octicon:`chevron-right`', + f'{status_badge(status, "accepted")}`Accepted`', + ':octicon:`kebab-horizontal`', + f'{status_badge(status, "rejected")}`Rejected`', + '\n', + f':octicon:`calendar` Proposal Date: {proposal_date}', + '\n', + ':octicon:`code-review` CL: ', + ', '.join([cl_link(cl_num) for cl_num in cl_nums]), + ] + ) + + card = CardDirective.create_card( + inst=self, + arguments=[title], + options={}, + ) + + return [card] + + +def setup(app: SphinxApplication): + app.add_directive('seed', PigweedSeedDirective) + + return { + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/pw_docgen/py/setup.cfg b/pw_docgen/py/setup.cfg index 41f7505ef..c139c9633 100644 --- a/pw_docgen/py/setup.cfg +++ b/pw_docgen/py/setup.cfg @@ -24,7 +24,6 @@ zip_safe = False install_requires = sphinx>=5.3.0 sphinx-argparse - sphinx-rtd-theme sphinxcontrib-mermaid>=0.7.1 sphinx-design>=0.3.0 diff --git a/pw_docgen/py/setup.py b/pw_docgen/py/setup.py deleted file mode 100644 index 02300488c..000000000 --- a/pw_docgen/py/setup.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2021 The Pigweed Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -"""pw_docgen""" - -import setuptools # type: ignore - -setuptools.setup() # Package definition in setup.cfg |