diff options
Diffstat (limited to 'pw_cli')
27 files changed, 711 insertions, 153 deletions
diff --git a/pw_cli/docs.rst b/pw_cli/docs.rst index b6dc32e3e..c4aa8ac0e 100644 --- a/pw_cli/docs.rst +++ b/pw_cli/docs.rst @@ -49,43 +49,56 @@ Here are some example invocations of ``pw``: Registering ``pw`` plugins ========================== Projects can register their own Python scripts as ``pw`` commands. ``pw`` -plugins are registered by providing the command name, module, and function in a -``PW_PLUGINS`` file. ``PW_PLUGINS`` files can add new commands or override -built-in commands. Since they are accessed by module name, plugins must be -defined in Python packages that are installed in the Pigweed virtual +plugins are registered by providing the command name, module, and function in +the ``pigweed.json`` file. ``pigweed.json`` files can add new commands or +override built-in commands. Since they are accessed by module name, plugins must +be defined in Python packages that are installed in the Pigweed virtual environment. -Plugin registrations in a ``PW_PLUGINS`` file apply to the their directory and -all subdirectories, similarly to configuration files like ``.clang-format``. -Registered plugins appear as commands in the ``pw`` tool when ``pw`` is run from -those directories. - -Projects that wish to register commands might place a ``PW_PLUGINS`` file in the -root of their repo. Multiple ``PW_PLUGINS`` files may be applied, but the ``pw`` -tool gives precedence to a ``PW_PLUGINS`` file in the current working directory -or the nearest parent directory. - -PW_PLUGINS file format ----------------------- -``PW_PLUGINS`` contains one plugin entry per line in the following format: - -.. code-block:: python - - # Lines that start with a # are ignored. - <command name> <Python module> <function> +pigweed.json file format +--------------------------- +``pigweed.json`` contains plugin entries in the following format: + +.. code-block:: + + { + "pw": { + "pw_cli": { + "plugins": { + "<plugin name>": { + "module": "<module containing plugin>", + "function": "<entry point for plugin>" + }, + ... + } + } + } + } The following example registers three commands: -.. code-block:: python - - # Register the presubmit script as pw presubmit - presubmit my_cool_project.tools run_presubmit - - # Override the pw test command with a custom version - test my_cool_project.testing run_test - - # Add a custom command - flash my_cool_project.flash main +.. code-block:: + + { + "pw": { + "pw_cli": { + "plugins": { + "presubmit": { + "module": "my_cool_project.tools", + "function": "run_presubmit" + }, + "test": { + "module": "my_cool_project.testing", + "function": "run_test" + }, + "flash": { + "module": "my_cool_project.flash", + "function": "main" + } + } + } + } + } Defining a plugin function -------------------------- @@ -280,50 +293,50 @@ registered (see :py:meth:`pw_cli.plugins.Registry.__init__`). Plugins may be registered in a few different ways. - * **Direct function call.** Register plugins by calling - :py:meth:`pw_cli.plugins.Registry.register` or - :py:meth:`pw_cli.plugins.Registry.register_by_name`. +* **Direct function call.** Register plugins by calling + :py:meth:`pw_cli.plugins.Registry.register` or + :py:meth:`pw_cli.plugins.Registry.register_by_name`. - .. code-block:: python + .. code-block:: python - registry = pw_cli.plugins.Registry() + registry = pw_cli.plugins.Registry() - registry.register('plugin_name', my_plugin) - registry.register_by_name('plugin_name', 'module_name', 'function_name') + registry.register('plugin_name', my_plugin) + registry.register_by_name('plugin_name', 'module_name', 'function_name') - * **Decorator.** Register using the :py:meth:`pw_cli.plugins.Registry.plugin` - decorator. +* **Decorator.** Register using the :py:meth:`pw_cli.plugins.Registry.plugin` + decorator. - .. code-block:: python + .. code-block:: python - _REGISTRY = pw_cli.plugins.Registry() + _REGISTRY = pw_cli.plugins.Registry() - # This function is registered as the "my_plugin" plugin. - @_REGISTRY.plugin - def my_plugin(): - pass + # This function is registered as the "my_plugin" plugin. + @_REGISTRY.plugin + def my_plugin(): + pass - # This function is registered as the "input" plugin. - @_REGISTRY.plugin(name='input') - def read_something(): - pass + # This function is registered as the "input" plugin. + @_REGISTRY.plugin(name='input') + def read_something(): + pass - The decorator may be aliased to give a cleaner syntax (e.g. ``register = - my_registry.plugin``). + The decorator may be aliased to give a cleaner syntax (e.g. ``register = + my_registry.plugin``). - * **Plugins files.** Plugins files use a simple format: +* **Plugins files.** Plugins files use a simple format: - .. code-block:: + .. code-block:: # Comments start with "#". Blank lines are ignored. name_of_the_plugin module.name module_member another_plugin some_module some_function - These files are placed in the file system and apply similarly to Git's - ``.gitignore`` files. From Python, these files are registered using - :py:meth:`pw_cli.plugins.Registry.register_file` and - :py:meth:`pw_cli.plugins.Registry.register_directory`. + These files are placed in the file system and apply similarly to Git's + ``.gitignore`` files. From Python, these files are registered using + :py:meth:`pw_cli.plugins.Registry.register_file` and + :py:meth:`pw_cli.plugins.Registry.register_directory`. pw_cli.plugins module reference ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/pw_cli/py/BUILD.gn b/pw_cli/py/BUILD.gn index 4bb4cd2e3..6b7ff68c4 100644 --- a/pw_cli/py/BUILD.gn +++ b/pw_cli/py/BUILD.gn @@ -15,12 +15,12 @@ import("//build_overrides/pigweed.gni") import("$dir_pw_build/python.gni") +import("$dir_pw_build/python_action_test.gni") pw_python_package("py") { setup = [ "pyproject.toml", "setup.cfg", - "setup.py", ] sources = [ "pw_cli/__init__.py", @@ -36,6 +36,10 @@ pw_python_package("py") { "pw_cli/process.py", "pw_cli/pw_command_plugins.py", "pw_cli/requires.py", + "pw_cli/shell_completion/__init__.py", + "pw_cli/shell_completion/zsh/__init__.py", + "pw_cli/shell_completion/zsh/pw/__init__.py", + "pw_cli/shell_completion/zsh/pw_build/__init__.py", "pw_cli/toml_config_loader_mixin.py", "pw_cli/yaml_config_loader_mixin.py", ] @@ -43,18 +47,22 @@ pw_python_package("py") { "envparse_test.py", "plugins_test.py", ] + python_deps = [ "$dir_pw_env_setup/py" ] pylintrc = "$dir_pigweed/.pylintrc" mypy_ini = "$dir_pigweed/.mypy.ini" + inputs = [ + "pw_cli/shell_completion/common.bash", + "pw_cli/shell_completion/pw.bash", + "pw_cli/shell_completion/pw.zsh", + "pw_cli/shell_completion/pw_build.bash", + "pw_cli/shell_completion/pw_build.zsh", + "pw_cli/shell_completion/zsh/pw/_pw", + "pw_cli/shell_completion/zsh/pw_build/_pw_build", + ] } -pw_python_script("process_integration_test") { +pw_python_action_test("process_integration_test") { sources = [ "process_integration_test.py" ] python_deps = [ ":py" ] - - pylintrc = "$dir_pigweed/.pylintrc" - mypy_ini = "$dir_pigweed/.mypy.ini" - - action = { - stamp = true - } + tags = [ "integration" ] } diff --git a/pw_cli/py/plugins_test.py b/pw_cli/py/plugins_test.py index a3f35132d..aadee9022 100644 --- a/pw_cli/py/plugins_test.py +++ b/pw_cli/py/plugins_test.py @@ -134,29 +134,6 @@ class TestPluginRegistry(unittest.TestCase): with self.assertRaises(plugins.Error): self._registry.register('bar', lambda: None) - def test_register_directory_innermost_takes_priority(self) -> None: - with tempfile.TemporaryDirectory() as tempdir: - paths = list(_create_files(tempdir, _TEST_PLUGINS)) - self._registry.register_directory(paths[1].parent, 'TEST_PLUGINS') - - self.assertEqual(self._registry.run_with_argv('test_plugin', []), 123) - - def test_register_directory_only_searches_up(self) -> None: - with tempfile.TemporaryDirectory() as tempdir: - paths = list(_create_files(tempdir, _TEST_PLUGINS)) - self._registry.register_directory(paths[0].parent, 'TEST_PLUGINS') - - self.assertEqual(self._registry.run_with_argv('test_plugin', []), 456) - - def test_register_directory_with_restriction(self) -> None: - with tempfile.TemporaryDirectory() as tempdir: - paths = list(_create_files(tempdir, _TEST_PLUGINS)) - self._registry.register_directory( - paths[0].parent, 'TEST_PLUGINS', Path(tempdir, 'nested', 'in') - ) - - self.assertNotIn('other_plugin', self._registry) - def test_register_same_file_multiple_times_no_error(self) -> None: with tempfile.TemporaryDirectory() as tempdir: paths = list(_create_files(tempdir, _TEST_PLUGINS)) diff --git a/pw_cli/py/process_integration_test.py b/pw_cli/py/process_integration_test.py index 3d05b1a1b..fa5873d30 100644 --- a/pw_cli/py/process_integration_test.py +++ b/pw_cli/py/process_integration_test.py @@ -21,8 +21,9 @@ from pw_cli import process import psutil # type: ignore - -FAST_TIMEOUT_SECONDS = 0.1 +# This timeout must be long enough to wait for the subprocess output, but +# fast enough that the test doesn't take terribly long in the success case. +FAST_TIMEOUT_SECONDS = 0.5 KILL_SIGNALS = set({-9, 137}) PYTHON = sys.executable @@ -44,8 +45,8 @@ class RunTest(unittest.TestCase): self.assertIn(result.returncode, KILL_SIGNALS) def test_timeout_kills_subprocess(self) -> None: - # Spawn a subprocess which waits for 100 seconds, print its pid, - # then wait for 100 seconds. + # Spawn a subprocess which prints its pid and then waits for 100 + # seconds. sleep_in_subprocess = textwrap.dedent( f""" import subprocess @@ -62,7 +63,7 @@ class RunTest(unittest.TestCase): PYTHON, '-c', sleep_in_subprocess, timeout=FAST_TIMEOUT_SECONDS ) self.assertIn(result.returncode, KILL_SIGNALS) - # THe first line of the output is the PID of the child sleep process. + # The first line of the output is the PID of the child sleep process. child_pid_str, sep, remainder = result.output.partition(b'\n') del sep, remainder child_pid = int(child_pid_str) diff --git a/pw_cli/py/pw_cli/__main__.py b/pw_cli/py/pw_cli/__main__.py index 3320a4556..363bc80a5 100644 --- a/pw_cli/py/pw_cli/__main__.py +++ b/pw_cli/py/pw_cli/__main__.py @@ -31,14 +31,33 @@ def main() -> NoReturn: pw_cli.log.install(level=args.loglevel, debug_log=args.debug_log) - # Start with the most critical part of the Pigweed command line tool. - if not args.no_banner: + # Print the banner unless --no-banner or --tab-complete-command is provided. + # Note: args.tab_complete_command may be the empty string '' so check for + # None instead. + if not args.no_banner and args.tab_complete_command is None: arguments.print_banner() _LOG.debug('Executing the pw command from %s', args.directory) os.chdir(args.directory) - pw_command_plugins.register(args.directory) + pw_command_plugins.register() + + if args.tab_complete_option is not None: + arguments.print_completions_for_option( + arguments.arg_parser(), + text=args.tab_complete_option, + tab_completion_format=args.tab_complete_format, + ) + sys.exit(0) + + if args.tab_complete_command is not None: + for name, plugin in sorted(pw_command_plugins.plugin_registry.items()): + if name.startswith(args.tab_complete_command): + if args.tab_complete_format == 'zsh': + print(':'.join([name, plugin.help()])) + else: + print(name) + sys.exit(0) if args.help or args.command is None: print(pw_command_plugins.format_help(), file=sys.stderr) diff --git a/pw_cli/py/pw_cli/arguments.py b/pw_cli/py/pw_cli/arguments.py index f7fe37b4e..a5b8e4b27 100644 --- a/pw_cli/py/pw_cli/arguments.py +++ b/pw_cli/py/pw_cli/arguments.py @@ -14,10 +14,12 @@ """Defines arguments for the pw command.""" import argparse +from dataclasses import dataclass, field +from enum import Enum import logging from pathlib import Path import sys -from typing import NoReturn +from typing import List, NoReturn, Optional from pw_cli import argument_types, plugins from pw_cli.branding import banner @@ -31,7 +33,77 @@ Example uses: def parse_args() -> argparse.Namespace: - return _parser().parse_args() + return arg_parser().parse_args() + + +class ShellCompletionFormat(Enum): + """Supported shell tab completion modes.""" + + BASH = 'bash' + ZSH = 'zsh' + + +@dataclass(frozen=True) +class ShellCompletion: + option_strings: List[str] = field(default_factory=list) + help: Optional[str] = None + choices: Optional[List[str]] = None + flag: bool = True + + def bash_completion(self, text: str) -> List[str]: + result: List[str] = [] + for option_str in self.option_strings: + if option_str.startswith(text): + result.append(option_str) + return result + + def zsh_completion(self, text: str) -> List[str]: + result: List[str] = [] + for option_str in self.option_strings: + if option_str.startswith(text): + short_and_long_opts = ' '.join(self.option_strings) + # '(-h --help)-h[Display help message and exit]' + # '(-h --help)--help[Display help message and exit]' + help_text = self.help if self.help else '' + state_str = '' + if not self.flag: + state_str = ': :->' + option_str + + result.append( + f'({short_and_long_opts}){option_str}[{help_text}]' + f'{state_str}' + ) + return result + + +def get_options_and_help( + parser: argparse.ArgumentParser, +) -> List[ShellCompletion]: + return list( + ShellCompletion( + option_strings=list(action.option_strings), + help=action.help, + choices=list(action.choices) if action.choices else [], + flag=action.nargs == 0, + ) + for action in parser._actions # pylint: disable=protected-access + ) + + +def print_completions_for_option( + parser: argparse.ArgumentParser, + text: str = '', + tab_completion_format: str = ShellCompletionFormat.BASH.value, +) -> None: + matched_lines: List[str] = [] + for completion in get_options_and_help(parser): + if tab_completion_format == ShellCompletionFormat.ZSH.value: + matched_lines.extend(completion.zsh_completion(text)) + else: + matched_lines.extend(completion.bash_completion(text)) + + for line in matched_lines: + print(line) def print_banner() -> None: @@ -41,7 +113,7 @@ def print_banner() -> None: def format_help(registry: plugins.Registry) -> str: """Returns the pw help information as a string.""" - return f'{_parser().format_help()}\n{registry.short_help()}' + return f'{arg_parser().format_help()}\n{registry.short_help()}' class _ArgumentParserWithBanner(argparse.ArgumentParser): @@ -53,7 +125,24 @@ class _ArgumentParserWithBanner(argparse.ArgumentParser): self.exit(2, f'{self.prog}: error: {message}\n') -def _parser() -> argparse.ArgumentParser: +def add_tab_complete_arguments( + parser: argparse.ArgumentParser, +) -> argparse.ArgumentParser: + parser.add_argument( + '--tab-complete-option', + nargs='?', + help='Print tab completions for the supplied option text.', + ) + parser.add_argument( + '--tab-complete-format', + choices=list(shell.value for shell in ShellCompletionFormat), + default='bash', + help='Output format for tab completion results.', + ) + return parser + + +def arg_parser() -> argparse.ArgumentParser: """Creates an argument parser for the pw command.""" argparser = _ArgumentParserWithBanner( prog='pw', @@ -97,6 +186,11 @@ def _parser() -> argparse.ArgumentParser: help='Do not print the Pigweed banner', ) argparser.add_argument( + '--tab-complete-command', + nargs='?', + help='Print tab completions for the supplied command text.', + ) + argparser.add_argument( 'command', nargs='?', help='Which command to run; see supported commands below', @@ -107,5 +201,4 @@ def _parser() -> argparse.ArgumentParser: nargs=argparse.REMAINDER, help='Remaining arguments are forwarded to the command', ) - - return argparser + return add_tab_complete_arguments(argparser) diff --git a/pw_cli/py/pw_cli/branding.py b/pw_cli/py/pw_cli/branding.py index 358e4019b..0d9dccd7f 100644 --- a/pw_cli/py/pw_cli/branding.py +++ b/pw_cli/py/pw_cli/branding.py @@ -42,7 +42,7 @@ def banner() -> str: # Take the banner from the file PW_BRANDING_BANNER; or use the default. banner_filename = parsed_env.PW_BRANDING_BANNER _memoized_banner = ( - Path(banner_filename).read_text() + Path(banner_filename).read_text(encoding='utf-8', errors='replace') if banner_filename else _PIGWEED_BANNER ) diff --git a/pw_cli/py/pw_cli/color.py b/pw_cli/py/pw_cli/color.py index 855a8d11c..6b9074bf5 100644 --- a/pw_cli/py/pw_cli/color.py +++ b/pw_cli/py/pw_cli/color.py @@ -68,13 +68,21 @@ def colors(enabled: Optional[bool] = None) -> Union[_Color, _NoColor]: """ if enabled is None: env = pw_cli.env.pigweed_environment() - enabled = env.PW_USE_COLOR or ( - sys.stdout.isatty() and sys.stderr.isatty() - ) + if 'PW_USE_COLOR' in os.environ: + enabled = env.PW_USE_COLOR + else: + enabled = sys.stdout.isatty() and sys.stderr.isatty() if enabled and os.name == 'nt': # Enable ANSI color codes in Windows cmd.exe. kernel32 = ctypes.windll.kernel32 # type: ignore kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7) + # These are semi-standard ways to turn colors off or on for many projects. + # See https://bixense.com/clicolors/ and https://no-color.org/ for more. + if 'NO_COLOR' in os.environ: + enabled = False + elif 'CLICOLOR_FORCE' in os.environ: + enabled = True + return _Color() if enabled else _NoColor() diff --git a/pw_cli/py/pw_cli/env.py b/pw_cli/py/pw_cli/env.py index 74deff8ea..afe3d8dfe 100644 --- a/pw_cli/py/pw_cli/env.py +++ b/pw_cli/py/pw_cli/env.py @@ -45,7 +45,7 @@ def pigweed_environment_parser() -> envparse.EnvironmentParser: ) parser.add_var('PW_SKIP_BOOTSTRAP') parser.add_var('PW_SUBPROCESS', type=envparse.strict_bool, default=False) - parser.add_var('PW_USE_COLOR', type=envparse.strict_bool, default=False) + parser.add_var('PW_USE_COLOR', type=envparse.strict_bool, default=True) parser.add_var('PW_USE_GCS_ENVSETUP', type=envparse.strict_bool) parser.add_allowed_suffix('_CIPD_INSTALL_DIR') @@ -69,6 +69,7 @@ def pigweed_environment_parser() -> envparse.EnvironmentParser: parser.add_var('PW_CONSOLE_CONFIG_FILE') parser.add_var('PW_ENVIRONMENT_NO_ERROR_ON_UNRECOGNIZED') + parser.add_var('PW_NO_CIPD_CACHE_DIR') parser.add_var('PW_CIPD_SERVICE_ACCOUNT_JSON') # RBE environment variables diff --git a/pw_cli/py/pw_cli/envparse.py b/pw_cli/py/pw_cli/envparse.py index d9ed9de47..182936f2d 100644 --- a/pw_cli/py/pw_cli/envparse.py +++ b/pw_cli/py/pw_cli/envparse.py @@ -22,6 +22,7 @@ from typing import ( Generic, IO, List, + Literal, Mapping, Optional, TypeVar, @@ -173,7 +174,10 @@ class EnvironmentParser: and var not in self._variables and not var.endswith(allowed_suffixes) ): - raise ValueError(f'Unrecognized environment variable {var}') + raise ValueError( + f'Unrecognized environment variable {var}, please ' + 'remove it from your environment' + ) return namespace @@ -202,9 +206,7 @@ def strict_bool(value: str) -> bool: ) -# TODO(mohrr) Switch to Literal when no longer supporting Python 3.7. -# OpenMode = Literal['r', 'rb', 'w', 'wb'] -OpenMode = str +OpenMode = Literal['r', 'rb', 'w', 'wb'] class FileType: diff --git a/pw_cli/py/pw_cli/log.py b/pw_cli/py/pw_cli/log.py index c8414d1f5..e1ef13fde 100644 --- a/pw_cli/py/pw_cli/log.py +++ b/pw_cli/py/pw_cli/log.py @@ -165,6 +165,12 @@ def install( 9, ): formatter.default_msec_format = '' + # For 3.8 set datefmt to time_format + elif sys.version_info >= ( + 3, + 8, + ): + formatter.datefmt = time_format # Set the log level on the root logger to NOTSET, so that all logs # propagated from child loggers are handled. diff --git a/pw_cli/py/pw_cli/plugins.py b/pw_cli/py/pw_cli/plugins.py index b82b7f675..15a5dcd2d 100644 --- a/pw_cli/py/pw_cli/plugins.py +++ b/pw_cli/py/pw_cli/plugins.py @@ -318,6 +318,33 @@ class Registry(collections.abc.Mapping): return plugin + def register_config( + self, + config: Dict, + path: Optional[Path] = None, + ) -> None: + """Registers plugins from a Pigweed config. + + Any exceptions raised from parsing the file are caught and logged. + """ + plugins = config.get('pw', {}).get('pw_cli', {}).get('plugins', {}) + for name, location in plugins.items(): + module = location.pop('module') + function = location.pop('function') + if location: + raise ValueError(f'unrecognized plugin options: {location}') + + try: + self.register_by_name(name, module, function, path) + except Error as err: + self._errors[name].append(err) + _LOG.error( + '%s Failed to register plugin "%s": %s', + path, + name, + err, + ) + def register_file(self, path: Path) -> None: """Registers plugins from a plugins file. diff --git a/pw_cli/py/pw_cli/process.py b/pw_cli/py/pw_cli/process.py index 98852e35f..c77627bd5 100644 --- a/pw_cli/py/pw_cli/process.py +++ b/pw_cli/py/pw_cli/process.py @@ -200,7 +200,8 @@ async def run_async( if process.returncode: _LOG.error('%s exited with status %d', program, process.returncode) else: - _LOG.error('%s exited successfully', program) + # process.returncode is 0 + _LOG.debug('%s exited successfully', program) return CompletedProcess(process, output) diff --git a/pw_cli/py/pw_cli/pw_command_plugins.py b/pw_cli/py/pw_cli/pw_command_plugins.py index 99b86fdd8..cfef10a42 100644 --- a/pw_cli/py/pw_cli/pw_command_plugins.py +++ b/pw_cli/py/pw_cli/pw_command_plugins.py @@ -14,14 +14,14 @@ """This module manages the global plugin registry for pw_cli.""" import argparse -import os from pathlib import Path import sys from typing import Iterable -from pw_cli import arguments, plugins +from pw_cli import arguments, env, plugins +import pw_env_setup.config_file -_plugin_registry = plugins.Registry(validator=plugins.callable_with_no_args) +plugin_registry = plugins.Registry(validator=plugins.callable_with_no_args) REGISTRY_FILE = 'PW_PLUGINS' @@ -31,6 +31,7 @@ def _register_builtin_plugins(registry: plugins.Registry) -> None: # Register these by name to avoid circular dependencies. registry.register_by_name('bloat', 'pw_bloat.__main__', 'main') registry.register_by_name('doctor', 'pw_doctor.doctor', 'main') + registry.register_by_name('emu', 'pw_emu.__main__', 'main') registry.register_by_name('format', 'pw_presubmit.format_code', 'main') registry.register_by_name('keep-sorted', 'pw_presubmit.keep_sorted', 'main') registry.register_by_name('logdemo', 'pw_cli.log', 'main') @@ -54,26 +55,33 @@ def _help_command(): help='command for which to display detailed info', ) - print(arguments.format_help(_plugin_registry), file=sys.stderr) + print(arguments.format_help(plugin_registry), file=sys.stderr) - for line in _plugin_registry.detailed_help(**vars(parser.parse_args())): + for line in plugin_registry.detailed_help(**vars(parser.parse_args())): print(line, file=sys.stderr) -def register(directory: Path) -> None: - _register_builtin_plugins(_plugin_registry) - _plugin_registry.register_directory( - directory, REGISTRY_FILE, Path(os.environ.get('PW_PROJECT_ROOT', '')) - ) +def register() -> None: + _register_builtin_plugins(plugin_registry) + parsed_env = env.pigweed_environment() + pw_plugins_file: Path = parsed_env.PW_PROJECT_ROOT / REGISTRY_FILE + + if pw_plugins_file.is_file(): + plugin_registry.register_file(pw_plugins_file) + else: + plugin_registry.register_config( + config=pw_env_setup.config_file.load(), + path=pw_env_setup.config_file.path(), + ) def errors() -> dict: - return _plugin_registry.errors() + return plugin_registry.errors() def format_help() -> str: - return arguments.format_help(_plugin_registry) + return arguments.format_help(plugin_registry) def run(name: str, args: Iterable[str]) -> int: - return _plugin_registry.run_with_argv(name, args) + return plugin_registry.run_with_argv(name, args) diff --git a/pw_cli/py/pw_cli/requires.py b/pw_cli/py/pw_cli/requires.py index 79a2b61de..d9a996b20 100755 --- a/pw_cli/py/pw_cli/requires.py +++ b/pw_cli/py/pw_cli/requires.py @@ -27,13 +27,16 @@ For more see http://go/pigweed-ci-cq-intro. """ import argparse +import dataclasses import json import logging +import os from pathlib import Path import re import subprocess import sys import tempfile +from typing import Callable, Dict, IO, List, Sequence import uuid HELPER_GERRIT = 'pigweed-internal' @@ -59,6 +62,42 @@ remote: _LOG = logging.getLogger(__name__) +@dataclasses.dataclass +class Change: + gerrit_name: str + number: int + + +class EnhancedJSONEncoder(json.JSONEncoder): + def default(self, o): + if dataclasses.is_dataclass(o): + return dataclasses.asdict(o) + return super().default(o) + + +def dump_json_patches(obj: Sequence[Change], outs: IO): + json.dump(obj, outs, indent=2, cls=EnhancedJSONEncoder) + + +def log_entry_exit(func: Callable) -> Callable: + def wrapper(*args, **kwargs): + _LOG.debug('entering %s()', func.__name__) + _LOG.debug('args %r', args) + _LOG.debug('kwargs %r', kwargs) + try: + res = func(*args, **kwargs) + _LOG.debug('return value %r', res) + return res + except Exception as exc: + _LOG.debug('exception %r', exc) + raise + finally: + _LOG.debug('exiting %s()', func.__name__) + + return wrapper + + +@log_entry_exit def parse_args() -> argparse.Namespace: """Creates an argument parser and parses arguments.""" @@ -78,7 +117,8 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def _run_command(*args, **kwargs): +@log_entry_exit +def _run_command(*args, **kwargs) -> subprocess.CompletedProcess: kwargs.setdefault('capture_output', True) _LOG.debug('%s', args) _LOG.debug('%s', kwargs) @@ -89,6 +129,7 @@ def _run_command(*args, **kwargs): return res +@log_entry_exit def check_status() -> bool: res = subprocess.run(['git', 'status'], capture_output=True) if res.returncode: @@ -97,34 +138,40 @@ def check_status() -> bool: return True +@log_entry_exit def clone(requires_dir: Path) -> None: _LOG.info('cloning helper repository into %s', requires_dir) _run_command(['git', 'clone', HELPER_REPO, '.'], cwd=requires_dir) -def create_commit(requires_dir: Path, requirements) -> None: +@log_entry_exit +def create_commit( + requires_dir: Path, requirement_strings: Sequence[str] +) -> None: """Create a commit in the local tree with the given requirements.""" change_id = str(uuid.uuid4()).replace('-', '00') _LOG.debug('change_id %s', change_id) - reqs = [] - for req in requirements: + requirement_objects: List[Change] = [] + for req in requirement_strings: gerrit_name, number = req.split(':', 1) - reqs.append({'gerrit_name': gerrit_name, 'number': number}) + requirement_objects.append(Change(gerrit_name, int(number))) path = requires_dir / 'patches.json' _LOG.debug('path %s', path) with open(path, 'w') as outs: - json.dump(reqs, outs) + dump_json_patches(requirement_objects, outs) + outs.write('\n') _run_command(['git', 'add', path], cwd=requires_dir) + # TODO: b/232234662 - Don't add 'Requires:' lines to commit messages. commit_message = [ f'{_DNS} {change_id[0:10]}\n\n', '', f'Change-Id: I{change_id}', ] - for req in requirements: + for req in requirement_strings: commit_message.append(f'Requires: {req}') _LOG.debug('message %s', commit_message) @@ -137,8 +184,18 @@ def create_commit(requires_dir: Path, requirements) -> None: _run_command(['git', 'show'], cwd=requires_dir) -def push_commit(requires_dir: Path, push=True) -> str: - output = DEFAULT_OUTPUT +@log_entry_exit +def push_commit(requires_dir: Path, push=True) -> Change: + """Push a commit to the helper repository. + + Args: + requires_dir: Local checkout of the helper repository. + push: Whether to actually push or if this is a local-only test. + + Returns a Change object referencing the pushed commit. + """ + + output: str = DEFAULT_OUTPUT if push: res = _run_command( ['git', 'push', HELPER_REPO, '+HEAD:refs/for/main'], @@ -157,22 +214,46 @@ def push_commit(requires_dir: Path, push=True) -> str: match = regex.search(output) if not match: raise ValueError(f"invalid output from 'git push': {output}") - change_num = match.group('num') + change_num = int(match.group('num')) _LOG.info('created %s change %s', HELPER_PROJECT, change_num) - return f'{HELPER_GERRIT}:{change_num}' + return Change(HELPER_GERRIT, change_num) + + +@log_entry_exit +def amend_existing_change(dependency: Dict[str, str]) -> None: + """Amend the current change to depend on the dependency + + Args: + dependency: The change on which the top of the current checkout now + depends. + """ + git_root = Path( + subprocess.run( + ['git', 'rev-parse', '--show-toplevel'], + capture_output=True, + ) + .stdout.decode() + .rstrip('\n') + ) + patches_json = git_root / 'patches.json' + _LOG.info('%s %d', patches_json, os.path.isfile(patches_json)) + patches = [] + if os.path.isfile(patches_json): + with open(patches_json, 'r') as ins: + patches = json.load(ins) -def amend_existing_change(change: str) -> None: - res = _run_command(['git', 'log', '-1', '--pretty=%B']) - original = res.stdout.rstrip().decode() + patches.append(dependency) + with open(patches_json, 'w') as outs: + dump_json_patches(patches, outs) + outs.write('\n') + _LOG.info('%s %d', patches_json, os.path.isfile(patches_json)) - addition = f'Requires: {change}' - _LOG.info('adding "%s" to current commit message', addition) - message = '\n'.join((original, addition)) - _run_command(['git', 'commit', '--amend', '--message', message]) + _run_command(['git', 'add', patches_json]) + _run_command(['git', 'commit', '--amend', '--no-edit']) -def run(requirements, push=True) -> int: +def run(requirements: Sequence[str], push: bool = True) -> int: """Entry point for requires.""" if not check_status(): diff --git a/pw_cli/py/pw_cli/shell_completion/__init__.py b/pw_cli/py/pw_cli/shell_completion/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/pw_cli/py/pw_cli/shell_completion/__init__.py diff --git a/pw_cli/py/pw_cli/shell_completion/common.bash b/pw_cli/py/pw_cli/shell_completion/common.bash new file mode 100644 index 000000000..95ed56171 --- /dev/null +++ b/pw_cli/py/pw_cli/shell_completion/common.bash @@ -0,0 +1,65 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +if [[ -n ${ZSH_VERSION-} ]]; then + autoload -U +X bashcompinit && bashcompinit +fi + +__pwcomp_words_include () +{ + local i=1 + while [[ $i -lt $COMP_CWORD ]]; do + if [[ "${COMP_WORDS[i]}" = "$1" ]]; then + return 0 + fi + i=$((++i)) + done + return 1 +} + +# Find the previous non-switch word +__pwcomp_prev () +{ + local idx=$((COMP_CWORD - 1)) + local prv="${COMP_WORDS[idx]}" + while [[ $prv == -* ]]; do + idx=$((--idx)) + prv="${COMP_WORDS[idx]}" + done +} + + +__pwcomp () +{ + # break $1 on space, tab, and newline characters, + # and turn it into a newline separated list of words + local list s sep=$'\n' IFS=$' '$'\t'$'\n' + local cur="${COMP_WORDS[COMP_CWORD]}" + + for s in $1; do + __pwcomp_words_include "$s" && continue + list="$list$s$sep" + done + + case "$cur" in + --*=) + COMPREPLY=() + ;; + *) + IFS=$sep + COMPREPLY=( $(compgen -W "$list" -- "$cur" | sed -e 's/[^=]$/& /g') ) + ;; + esac +} + diff --git a/pw_cli/py/pw_cli/shell_completion/pw.bash b/pw_cli/py/pw_cli/shell_completion/pw.bash new file mode 100644 index 000000000..a32085395 --- /dev/null +++ b/pw_cli/py/pw_cli/shell_completion/pw.bash @@ -0,0 +1,75 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# Source common bash completion functions +# Path to this directory, works in bash and zsh +COMPLETION_DIR=$(dirname "${BASH_SOURCE[0]-$0}") +. "${COMPLETION_DIR}/common.bash" + +_pw () { + local i=1 cmd + + if [[ -n ${ZSH_VERSION-} ]]; then + emulate -L bash + setopt KSH_TYPESET + + # Workaround zsh's bug that leaves 'words' as a special + # variable in versions < 4.3.12 + typeset -h words + fi + + # find the subcommand + while [[ $i -lt $COMP_CWORD ]]; do + local s="${COMP_WORDS[i]}" + case "$s" in + --*) ;; + -*) ;; + *) cmd="$s" + break + ;; + esac + i=$((++i)) + done + + if [[ $i -eq $COMP_CWORD ]]; then + local cur="${COMP_WORDS[COMP_CWORD]}" + case "$cur" in + -*) + local all_options=$(pw --no-banner --tab-complete-option "") + __pwcomp "${all_options}" + return + ;; + *) + local all_commands=$(pw --no-banner --tab-complete-command "") + __pwcomp "${all_commands}" + return + ;; + esac + return + fi + + # subcommands have their own completion functions + case "$cmd" in + help) + local all_commands=$(pw --no-banner --tab-complete-command "") + __pwcomp "${all_commands}" + ;; + *) + # If the command is 'build' and a function named _pw_build exists, then run it. + [[ $(type -t _pw_$cmd) == function ]] && _pw_$cmd + ;; + esac +} + +complete -o bashdefault -o default -o nospace -F _pw pw diff --git a/pw_cli/py/setup.py b/pw_cli/py/pw_cli/shell_completion/pw.zsh index 73ab7458c..ff70ab797 100644 --- a/pw_cli/py/setup.py +++ b/pw_cli/py/pw_cli/shell_completion/pw.zsh @@ -1,4 +1,4 @@ -# Copyright 2021 The Pigweed Authors +# Copyright 2023 The Pigweed Authors # # Licensed under the Apache License, Version 2.0 (the "License"); you may not # use this file except in compliance with the License. You may obtain a copy of @@ -11,8 +11,6 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. -"""pw_cli""" -import setuptools # type: ignore - -setuptools.setup() # Package definition in setup.cfg +fpath=("$PW_ROOT/pw_cli/py/pw_cli/shell_completion/zsh/pw" $fpath) +autoload -Uz compinit && compinit -i diff --git a/pw_cli/py/pw_cli/shell_completion/pw_build.bash b/pw_cli/py/pw_cli/shell_completion/pw_build.bash new file mode 100644 index 000000000..3a0c3ded5 --- /dev/null +++ b/pw_cli/py/pw_cli/shell_completion/pw_build.bash @@ -0,0 +1,62 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# Source common bash completion functions +# Path to this directory, works in bash and zsh + +__pw_complete_recipes () { + local cur="${COMP_WORDS[COMP_CWORD]}" + local all_recipes=$(pw --no-banner build --tab-complete-recipe "${cur}") + COMPREPLY=($(compgen -W "$all_recipes" -- "$cur")) +} + +__pw_complete_presubmit_steps () { + local cur="${COMP_WORDS[COMP_CWORD]}" + local all_recipes=$(pw --no-banner build --tab-complete-presubmit-step "${cur}") + COMPREPLY=($(compgen -W "$all_recipes" -- "$cur")) +} + +_pw_build () { + local cur="${COMP_WORDS[COMP_CWORD]}" + local prev="${COMP_WORDS[COMP_CWORD-1]}" + + case "$prev" in + -r|--recipe) + __pw_complete_recipes + return + ;; + -s|--step) + __pw_complete_presubmit_steps + return + ;; + --logfile) + # Complete a file + COMPREPLY=($(compgen -f "$cur")) + return + ;; + *) + ;; + esac + + case "$cur" in + -*) + __pwcomp + local all_options=$(pw --no-banner build --tab-complete-option "") + __pwcomp "${all_options}" + return + ;; + esac + # Non-option args go here + COMPREPLY=() +} diff --git a/pw_cli/py/pw_cli/shell_completion/pw_build.zsh b/pw_cli/py/pw_cli/shell_completion/pw_build.zsh new file mode 100644 index 000000000..d2e4e75a6 --- /dev/null +++ b/pw_cli/py/pw_cli/shell_completion/pw_build.zsh @@ -0,0 +1,16 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +fpath=("$PW_ROOT/pw_cli/py/pw_cli/shell_completion/zsh/pw_build" $fpath) +autoload -Uz compinit && compinit -i diff --git a/pw_cli/py/pw_cli/shell_completion/zsh/__init__.py b/pw_cli/py/pw_cli/shell_completion/zsh/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/pw_cli/py/pw_cli/shell_completion/zsh/__init__.py diff --git a/pw_cli/py/pw_cli/shell_completion/zsh/pw/__init__.py b/pw_cli/py/pw_cli/shell_completion/zsh/pw/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/pw_cli/py/pw_cli/shell_completion/zsh/pw/__init__.py diff --git a/pw_cli/py/pw_cli/shell_completion/zsh/pw/_pw b/pw_cli/py/pw_cli/shell_completion/zsh/pw/_pw new file mode 100755 index 000000000..1b077763f --- /dev/null +++ b/pw_cli/py/pw_cli/shell_completion/zsh/pw/_pw @@ -0,0 +1,49 @@ +#compdef pw +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +_complete_pw_subcommands() { + local -a _1st_arguments=("${(@f)$(pw --no-banner --tab-complete-format=zsh --tab-complete-command '')}") + _describe -t commands "pw subcommand" _1st_arguments +} + +_pw() { + # Complete pw options + local -a _pw_options=("${(@f)$(pw --no-banner --tab-complete-format=zsh --tab-complete-option '')}") + _pw_options+=('*:: :->subcmds') + _arguments $_pw_options && return 0 + + # Match Sub-command + if (( CURRENT == 1 )); then + _complete_pw_subcommands + return + fi + + # Completion for each sub command + case "$words[1]" in + help) + # Help takes a subcommand name to display help for + _complete_pw_subcommands + ;; + *) + # If the command is 'build' and a function named _pw_build exists, then run it. + (( $+functions[_pw_$words[1]] )) && _pw_$words[1] && return 0 + # Otherwise, complete files. + _files + ;; + + esac +} + +_pw diff --git a/pw_cli/py/pw_cli/shell_completion/zsh/pw_build/__init__.py b/pw_cli/py/pw_cli/shell_completion/zsh/pw_build/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/pw_cli/py/pw_cli/shell_completion/zsh/pw_build/__init__.py diff --git a/pw_cli/py/pw_cli/shell_completion/zsh/pw_build/_pw_build b/pw_cli/py/pw_cli/shell_completion/zsh/pw_build/_pw_build new file mode 100644 index 000000000..ecaf8124e --- /dev/null +++ b/pw_cli/py/pw_cli/shell_completion/zsh/pw_build/_pw_build @@ -0,0 +1,40 @@ +#autoload +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +_pw_build() { + local -a _pw_build_options=("${(@f)$(pw --no-banner build --tab-complete-format=zsh --tab-complete-option '')}") + _arguments $_pw_build_options && return 0 + + case "$state" in + -r|--recipe) + _pw_build_recipe + ;; + -s|--step) + _pw_build_presubmit_step + ;; + *) + ;; + esac +} + +_pw_build_recipe() { + local -a all_recipes=("${(@f)$(pw --no-banner build --tab-complete-recipe '')}") + _values recipe $all_recipes +} + +_pw_build_presubmit_step() { + local -a all_steps=("${(@f)$(pw --no-banner build --tab-complete-presubmit-step '')}") + _values 'presubmit step' $all_steps +} diff --git a/pw_cli/py/setup.cfg b/pw_cli/py/setup.cfg index 20af8de6f..290b0a364 100644 --- a/pw_cli/py/setup.cfg +++ b/pw_cli/py/setup.cfg @@ -30,4 +30,12 @@ install_requires = console_scripts = pw = pw_cli.__main__:main [options.package_data] -pw_cli = py.typed +pw_cli = + py.typed + shell_completion/common.bash + shell_completion/pw.bash + shell_completion/pw.zsh + shell_completion/pw_build.bash + shell_completion/pw_build.zsh + shell_completion/zsh/pw/_pw + shell_completion/zsh/pw_build/_pw_build |