diff options
author | Rob Mohr <mohrr@google.com> | 2020-10-22 11:10:24 -0700 |
---|---|---|
committer | CQ Bot Account <pigweed-scoped@luci-project-accounts.iam.gserviceaccount.com> | 2020-10-22 21:12:03 +0000 |
commit | 0b6a5021623988f7720c266a5344de5aff0a6618 (patch) | |
tree | d18dcfdcaec0acc055eb6ec2970ba5ebc93fe7bf /pw_package | |
parent | 407bdad920ab8c9542eefa01b366d7151d3c13cc (diff) | |
download | pigweed-0b6a5021623988f7720c266a5344de5aff0a6618.tar.gz |
pw_package: Initial commit
Add pw_package module. This manages dependencies that aren't pulled in
through env setup. For now only nanopb is available through pw_package.
Change-Id: Ib8a20102baf27d5964bb275088c265f9334b6ff3
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/22020
Reviewed-by: Anthony DiGirolamo <tonymd@google.com>
Reviewed-by: Keir Mierle <keir@google.com>
Commit-Queue: Rob Mohr <mohrr@google.com>
Diffstat (limited to 'pw_package')
-rw-r--r-- | pw_package/BUILD.gn | 21 | ||||
-rw-r--r-- | pw_package/docs.rst | 110 | ||||
-rw-r--r-- | pw_package/py/pw_package/git_repo.py | 71 | ||||
-rw-r--r-- | pw_package/py/pw_package/package_manager.py | 147 | ||||
-rw-r--r-- | pw_package/py/pw_package/packages/__init__.py | 13 | ||||
-rw-r--r-- | pw_package/py/pw_package/packages/nanopb.py | 30 | ||||
-rw-r--r-- | pw_package/py/pw_package/pigweed_packages.py | 29 | ||||
-rw-r--r-- | pw_package/py/setup.py | 26 |
8 files changed, 447 insertions, 0 deletions
diff --git a/pw_package/BUILD.gn b/pw_package/BUILD.gn new file mode 100644 index 000000000..dd021e82f --- /dev/null +++ b/pw_package/BUILD.gn @@ -0,0 +1,21 @@ +# Copyright 2020 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import("//build_overrides/pigweed.gni") + +import("$dir_pw_docgen/docs.gni") + +pw_doc_group("docs") { + sources = [ "docs.rst" ] +} diff --git a/pw_package/docs.rst b/pw_package/docs.rst new file mode 100644 index 000000000..b3055db97 --- /dev/null +++ b/pw_package/docs.rst @@ -0,0 +1,110 @@ +.. _module-pw_package: + +========== +pw_package +========== +The package module provides a mechanism to install additional tools used by +Pigweed. Most Pigweed dependencies should be installed using +:ref:`module-pw_env_setup`. Examples of reasons packages should be managed using +this module instead are listed below. + +* The dependency is extremely large and not commonly used. +* The dependency has a number of compatible versions and we want to allow + downstream projects to pick a version rather than being forced to use ours. +* The dependency has license issues that make it complicated for Google to + include it directly as a submodule or distribute it as a CIPD package. +* The dependency needs to be "installed" into the system in some manner beyond + just extraction and thus isn't a good match for distribution with CIPD. + +----- +Usage +----- +The package module can be accessed through the ``pw package`` command. This +has several subcommands. + +``pw package list`` + Lists all the packages installed followed by all the packages available. + +``pw package install <package-name>`` + Installs ``<package-name>``. Exactly how this works is package-dependent, + and packages can decide to do nothing because the package is current, do an + incremental update, or delete the current version and install anew. Use + ``--force`` to remove the package before installing. + +``pw package status <package-name>`` + Indicates whether ``<packagxe-name>`` is installed. + +``pw package remove <package-name>`` + Removes ``<package-name>``. + +----------- +Configuring +----------- + +Compatibility +~~~~~~~~~~~~~ +Python 3 + +Adding a New Package +~~~~~~~~~~~~~~~~~~~~ +To add a new package create a class that subclasses ``Package`` from +``pw_package/package_manager.py``. + +.. code-block:: python + + class Package: + """Package to be installed. + + Subclass this to implement installation of a specific package. + """ + def __init__(self, name): + self._name = name + + @property + def name(self): + return self._name + + def install(self, path: pathlib.Path) -> None: + """Install the package at path. + + Install the package in path. Cannot assume this directory is empty—it + may need to be deleted or updated. + """ + + def remove(self, path: pathlib.Path) -> None: + """Remove the package from path. + + Removes the directory containing the package. For most packages this + should be sufficient to remove the package, and subclasses should not + need to override this package. + """ + if os.path.exists(path): + shutil.rmtree(path) + + def status(self, path: pathlib.Path) -> bool: + """Returns if package is installed at path and current. + + This method will be skipped if the directory does not exist. + """ + +There's also a helper class for retrieving specific revisions of Git +repositories in ``pw_package/git_repo.py``. + +Then call ``pw_package.package_manager.register(PackageClass)`` to register +the class with the package manager. + +Setting up a Project +~~~~~~~~~~~~~~~~~~~~ +To set up the package manager for a new project create a file like below and +add it to the ``PW_PLUGINS`` file (see :ref:`module-pw_cli` for details). This +file is based off of ``pw_package/pigweed_packages.py``. + +.. code-block:: python + + from pw_package import package_manager + # These modules register themselves so must be imported despite appearing + # unused. + from pw_package.packages import nanopb + + def main(argv=None) -> int: + return package_manager.run(**vars(package_manager.parse_args(argv))) diff --git a/pw_package/py/pw_package/git_repo.py b/pw_package/py/pw_package/git_repo.py new file mode 100644 index 000000000..4b983663a --- /dev/null +++ b/pw_package/py/pw_package/git_repo.py @@ -0,0 +1,71 @@ +# Copyright 2020 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Install and check status of Git repository-based packages.""" + +import os +import pathlib +import shutil +import subprocess +from typing import Union + +import pw_package.package_manager + +PathOrStr = Union[pathlib.Path, str] + + +def git_stdout(*args: PathOrStr, + show_stderr=False, + repo: PathOrStr = '.') -> str: + return subprocess.run(['git', '-C', repo, *args], + stdout=subprocess.PIPE, + stderr=None if show_stderr else subprocess.DEVNULL, + check=True).stdout.decode().strip() + + +def git(*args: PathOrStr, + repo: PathOrStr = '.') -> subprocess.CompletedProcess: + return subprocess.run(['git', '-C', repo, *args], check=True) + + +class GitRepo(pw_package.package_manager.Package): + """Install and check status of Git repository-based packages.""" + def __init__(self, url, commit, *args, **kwargs): + super().__init__(*args, **kwargs) + self._url = url + self._commit = commit + + def status(self, path: pathlib.Path) -> bool: + if not os.path.isdir(path / '.git'): + return False + + remote = git_stdout('remote', 'get-url', 'origin', repo=path) + commit = git_stdout('rev-parse', 'HEAD', repo=path) + status = git_stdout('status', '--porcelain=v1', repo=path) + return remote == self._url and commit == self._commit and not status + + def install(self, path: pathlib.Path) -> None: + # If already installed and at correct version exit now. + if self.status(path): + return + + # Otherwise delete current version and clone again. + if os.path.isdir(path): + shutil.rmtree(path) + + # --filter=blob:none means we don't get history, just the current + # revision. If we later run commands that need history it will be + # retrieved on-demand. For small repositories the effect is negligible + # but for large repositories this should be a significant improvement. + git('clone', '--filter=blob:none', self._url, path) + git('reset', '--hard', self._commit, repo=path) diff --git a/pw_package/py/pw_package/package_manager.py b/pw_package/py/pw_package/package_manager.py new file mode 100644 index 000000000..dd20b44d2 --- /dev/null +++ b/pw_package/py/pw_package/package_manager.py @@ -0,0 +1,147 @@ +# Copyright 2020 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Install and remove optional packages.""" + +import argparse +import logging +import os +import pathlib +import shutil +from typing import List + +_LOG: logging.Logger = logging.getLogger(__name__) + + +class Package: + """Package to be installed. + + Subclass this to implement installation of a specific package. + """ + def __init__(self, name): + self._name = name + + @property + def name(self): + return self._name + + def install(self, path: pathlib.Path) -> None: # pylint: disable=no-self-use + """Install the package at path. + + Install the package in path. Cannot assume this directory is empty—it + may need to be deleted or updated. + """ + + def remove(self, path: pathlib.Path) -> None: # pylint: disable=no-self-use + """Remove the package from path. + + Removes the directory containing the package. For most packages this + should be sufficient to remove the package, and subclasses should not + need to override this package. + """ + if os.path.exists(path): + shutil.rmtree(path) + + def status(self, path: pathlib.Path) -> bool: # pylint: disable=no-self-use + """Returns if package is installed at path and current. + + This method will be skipped if the directory does not exist. + """ + + +_PACKAGES = {} + + +def register(package_class: type) -> None: + obj = package_class() + _PACKAGES[obj.name] = obj + + +class PackageManager: + """Install and remove optional packages.""" + def __init__(self): + self._pkg_root: pathlib.Path = None + + def install(self, package: str, force=False): + pkg = _PACKAGES[package] + if force: + self.remove(package) + _LOG.info('Installing %s...', pkg.name) + pkg.install(self._pkg_root / pkg.name) + _LOG.info('Installing %s...done.', pkg.name) + return 0 + + def remove(self, package: str): # pylint: disable=no-self-use + pkg = _PACKAGES[package] + _LOG.info('Removing %s...', pkg.name) + pkg.remove(self._pkg_root / pkg.name) + _LOG.info('Removing %s...done.', pkg.name) + return 0 + + def status(self, package: str): # pylint: disable=no-self-use + pkg = _PACKAGES[package] + path = self._pkg_root / pkg.name + if os.path.isdir(path) and pkg.status(path): + _LOG.info('%s is installed.', pkg.name) + return 0 + + _LOG.info('%s is not installed.', pkg.name) + return -1 + + def list(self): # pylint: disable=no-self-use + _LOG.info('Installed packages:') + available = [] + for package in sorted(_PACKAGES.keys()): + pkg = _PACKAGES[package] + if pkg.status(self._pkg_root / pkg.name): + _LOG.info(' %s', pkg.name) + else: + available.append(pkg.name) + _LOG.info('') + + _LOG.info('Available packages:') + for pkg_name in available: + _LOG.info(' %s', pkg_name) + _LOG.info('') + + return 0 + + def run(self, command: str, pkg_root: pathlib.Path, **kwargs): + os.makedirs(pkg_root, exist_ok=True) + self._pkg_root = pkg_root + return getattr(self, command)(**kwargs) + + +def parse_args(argv: List[str] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser("Manage packages.") + parser.add_argument( + '--package-root', + '-e', + dest='pkg_root', + type=pathlib.Path, + default=(pathlib.Path(os.environ['_PW_ACTUAL_ENVIRONMENT_ROOT']) / + 'packages'), + ) + subparsers = parser.add_subparsers(dest='command', required=True) + install = subparsers.add_parser('install') + install.add_argument('--force', '-f', action='store_true') + remove = subparsers.add_parser('remove') + status = subparsers.add_parser('status') + for cmd in (install, remove, status): + cmd.add_argument('package', choices=_PACKAGES.keys()) + _ = subparsers.add_parser('list') + return parser.parse_args(argv) + + +def run(**kwargs): + return PackageManager().run(**kwargs) diff --git a/pw_package/py/pw_package/packages/__init__.py b/pw_package/py/pw_package/packages/__init__.py new file mode 100644 index 000000000..2c8334fad --- /dev/null +++ b/pw_package/py/pw_package/packages/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. diff --git a/pw_package/py/pw_package/packages/nanopb.py b/pw_package/py/pw_package/packages/nanopb.py new file mode 100644 index 000000000..96955bde4 --- /dev/null +++ b/pw_package/py/pw_package/packages/nanopb.py @@ -0,0 +1,30 @@ +# Copyright 2020 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Install and check status of nanopb.""" + +import pw_package.git_repo +import pw_package.package_manager + + +class NanoPB(pw_package.git_repo.GitRepo): + """Install and check status of nanopb.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, + name='nanopb', + url='https://github.com/nanopb/nanopb.git', + commit='9f57cc871d8a025039019c2d2fde217591f4e30d', + **kwargs) + + +pw_package.package_manager.register(NanoPB) diff --git a/pw_package/py/pw_package/pigweed_packages.py b/pw_package/py/pw_package/pigweed_packages.py new file mode 100644 index 000000000..734b6d7cb --- /dev/null +++ b/pw_package/py/pw_package/pigweed_packages.py @@ -0,0 +1,29 @@ +# Copyright 2020 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Install and remove optional packages for Pigweed.""" + +import sys + +from pw_package import package_manager +# These modules register themselves so must be imported despite appearing +# unused. +from pw_package.packages import nanopb # pylint: disable=unused-import + + +def main(argv=None) -> int: + return package_manager.run(**vars(package_manager.parse_args(argv))) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/pw_package/py/setup.py b/pw_package/py/setup.py new file mode 100644 index 000000000..682e0130f --- /dev/null +++ b/pw_package/py/setup.py @@ -0,0 +1,26 @@ +# Copyright 2020 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""The pw_package package.""" + +import setuptools + +setuptools.setup( + name='pw_package', + version='0.0.1', + author='Pigweed Authors', + author_email='pigweed-developers@googlegroups.com', + description='Tools for installing optional packages', + install_requires=[], + packages=setuptools.find_packages(), +) |