diff options
329 files changed, 10574 insertions, 3114 deletions
diff --git a/.bazelignore b/.bazelignore index 280ad54fd..5b3b096ae 100644 --- a/.bazelignore +++ b/.bazelignore @@ -4,3 +4,8 @@ environment node_modules out +bazel-bin +bazel-out +bazel-pigweed +bazel-testlogs +outbazel diff --git a/.gitignore b/.gitignore index f06c43cfc..ed3ae360b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ compile_commands.json out/ bazel-* +outbazel/ .presubmit/ docs/_build @@ -1,10 +1,5 @@ [MASTER] # inclusive-language: ignore -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code. -extension-pkg-allowlist=mypy - # Add files or directories to the blocklist. They should be base names, not # paths. ignore=CVS @@ -28,7 +23,7 @@ limit-inference-results=100 # List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. -load-plugins= +load-plugins=pylint.extensions.no_self_use # Pickle collected data for later comparisons. persistent=yes @@ -60,11 +55,23 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable=bad-continuation, # Rely on yapf for formatting +disable=broad-exception-raised, + consider-iterating-dictionary, + consider-using-f-string, + consider-using-generator, + consider-using-in, consider-using-with, fixme, - subprocess-run-check, + implicit-str-concat, raise-missing-from, + redundant-u-string-prefix, + subprocess-run-check, + superfluous-parens, + unnecessary-lambda-assignment, + unspecified-encoding, + use-dict-literal, + use-list-literal, + # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option @@ -131,13 +138,6 @@ max-line-length=80 # Maximum number of lines in a module. max-module-lines=9999 -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma, - dict-separator - # Allow the body of a class to be on the same line as the declaration if body # contains single statement. single-line-class-stmt=no @@ -161,7 +161,7 @@ allow-global-unused-variables=yes callbacks=cb_, _cb -# A regular expression matching the name of placeholder variables (i.e. +# A regular expression matching the name of placeholder variables (i.e. # expected to not be used). # inclusive-language: ignore dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ @@ -309,7 +309,7 @@ good-names=i, _ # Include a hint for the correct naming format with invalid-name. -include-naming-hint=no +include-naming-hint=yes # Naming style matching correct inline iteration names. inlinevar-naming-style=any @@ -511,5 +511,5 @@ preferred-modules= # Exceptions that will emit a warning when being caught. Defaults to # "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception +overgeneral-exceptions=builtins.BaseException, + builtins.Exception @@ -86,7 +86,6 @@ assert(pw_toolchain_SUPPORTED_C_OPTIMIZATION_LEVELS + # exclusively to facilitate easy upstream development and testing. group("default") { deps = [ - ":check_modules", ":docs", ":host", ":pi_pico", @@ -94,6 +93,7 @@ group("default") { ":python.tests", ":static_analysis", ":stm32f429i", + ":warn_if_modules_out_of_date", ] } @@ -126,11 +126,22 @@ pool("module_check_pool") { depth = 1 } -# Check that PIGWEED_MODULES is up-to-date and sorted. -action("check_modules") { +# Warns if PIGWEED_MODULES is not up-to-date and sorted. +action("warn_if_modules_out_of_date") { forward_variables_from(_update_or_check_modules_lists, "*") outputs = [ "$target_gen_dir/$target_name.passed" ] - args += [ "--warn-only" ] + rebase_path(outputs, root_build_dir) + args += [ + "--mode=WARN", + "--stamp", + ] + rebase_path(outputs, root_build_dir) + pool = ":module_check_pool" +} + +# Fails if PIGWEED_MODULES is not up-to-date and sorted. +action("check_modules") { + forward_variables_from(_update_or_check_modules_lists, "*") + outputs = [ "$target_gen_dir/$target_name.ALWAYS_RERUN" ] # Never created + args += [ "--mode=CHECK" ] pool = ":module_check_pool" } @@ -139,6 +150,7 @@ action("check_modules") { action("update_modules") { forward_variables_from(_update_or_check_modules_lists, "*") outputs = [ "$target_gen_dir/$target_name.ALWAYS_RERUN" ] # Never created + args += [ "--mode=UPDATE" ] pool = ":module_check_pool" } @@ -266,17 +278,33 @@ group("action_tests") { group("integration_tests") { _default_tc = _default_toolchain_prefix + pw_DEFAULT_C_OPTIMIZATION_LEVEL deps = [ + "$dir_pw_cli/py:process_integration_test.action($_default_tc)", "$dir_pw_rpc:cpp_client_server_integration_test($_default_tc)", "$dir_pw_rpc/py:python_client_cpp_server_test.action($_default_tc)", "$dir_pw_unit_test/py:rpc_service_test.action($_default_tc)", ] } -# OSS-Fuzz uses this target to build fuzzers alone. +# Build-only target for fuzzers. group("fuzzers") { - # Fuzzing is only supported on Linux and MacOS using clang. + deps = [] + + # TODO(b/274437709): The client_fuzzer encounters build errors on macos. Limit + # it to Linux hosts for now. + if (host_os == "linux") { + _default_tc = _default_toolchain_prefix + pw_DEFAULT_C_OPTIMIZATION_LEVEL + deps += [ "$dir_pw_rpc/fuzz:client_fuzzer($_default_tc)" ] + } + if (host_os != "win") { - deps = [ ":pw_module_tests($dir_pigweed/targets/host:host_clang_fuzz)" ] + # Coverage-guided fuzzing is only supported on Linux and MacOS using clang. + deps += [ + "$dir_pw_bluetooth_hci:fuzzers($dir_pigweed/targets/host:host_clang_fuzz)", + "$dir_pw_fuzzer:fuzzers($dir_pigweed/targets/host:host_clang_fuzz)", + "$dir_pw_protobuf:fuzzers($dir_pigweed/targets/host:host_clang_fuzz)", + "$dir_pw_random:fuzzers($dir_pigweed/targets/host:host_clang_fuzz)", + "$dir_pw_tokenizer:fuzzers($dir_pigweed/targets/host:host_clang_fuzz)", + ] } } diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c5266a08..4e463d29a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,8 +29,14 @@ if(CONFIG_ZEPHYR_PIGWEED_MODULE) pw_chrono.system_clock pw_chrono_zephyr.system_clock pw_chrono/backend.cmake) pw_set_zephyr_backend_ifdef(CONFIG_PIGWEED_INTERRUPT_CONTEXT pw_interrupt.context pw_interrupt_zephyr.context pw_interrupt/backend.cmake) - pw_set_zephyr_backend_ifdef(CONFIG_PIGWEED_LOG + pw_set_zephyr_backend_ifdef(CONFIG_PIGWEED_LOG_ZEPHYR pw_log pw_log_zephyr pw_log/backend.cmake) + pw_set_zephyr_backend_ifdef(CONFIG_PIGWEED_LOG_TOKENIZED + pw_log_tokenized.handler pw_log_zephyr.tokenized_handler pw_log_tokenized/backend.cmake) + pw_set_zephyr_backend_ifdef(CONFIG_PIGWEED_LOG_TOKENIZED + pw_log pw_log_tokenized pw_log/backend.cmake) + pw_set_zephyr_backend_ifdef(CONFIG_PIGWEED_THREAD_SLEEP + pw_thread.sleep pw_thread_zephyr.sleep pw_thread/backend.cmake) pw_set_zephyr_backend_ifdef(CONFIG_PIGWEED_SYNC_MUTEX pw_sync.mutex pw_sync_zephyr.mutex_backend pw_sync/backend.cmake) pw_set_zephyr_backend_ifdef(CONFIG_PIGWEED_SYNC_BINARY_SEMAPHORE @@ -110,6 +116,7 @@ add_subdirectory(pw_system EXCLUDE_FROM_ALL) add_subdirectory(pw_thread EXCLUDE_FROM_ALL) add_subdirectory(pw_thread_freertos EXCLUDE_FROM_ALL) add_subdirectory(pw_thread_stl EXCLUDE_FROM_ALL) +add_subdirectory(pw_thread_zephyr EXCLUDE_FROM_ALL) add_subdirectory(pw_tokenizer EXCLUDE_FROM_ALL) add_subdirectory(pw_toolchain EXCLUDE_FROM_ALL) add_subdirectory(pw_trace EXCLUDE_FROM_ALL) diff --git a/Kconfig.zephyr b/Kconfig.zephyr index afb0c6872..9e07bf0ae 100644 --- a/Kconfig.zephyr +++ b/Kconfig.zephyr @@ -13,7 +13,7 @@ # the License. config ZEPHYR_PIGWEED_MODULE - select LIB_CPLUSPLUS + select REQUIRES_FULL_LIBCPP depends on STD_CPP17 || STD_CPP2A || STD_CPP20 || STD_CPP2B if ZEPHYR_PIGWEED_MODULE @@ -38,6 +38,7 @@ rsource "pw_stream/Kconfig" rsource "pw_string/Kconfig" rsource "pw_sync_zephyr/Kconfig" rsource "pw_sys_io_zephyr/Kconfig" +rsource "pw_thread_zephyr/Kconfig" rsource "pw_tokenizer/Kconfig" rsource "pw_varint/Kconfig" @@ -1,4 +1,4 @@ # CHRE team maintainers in AOSP bduddie@google.com -karthikmb@google.com +berchet@google.com stange@google.com diff --git a/PIGWEED_MODULES b/PIGWEED_MODULES index a75a246e2..ef740fcf3 100644 --- a/PIGWEED_MODULES +++ b/PIGWEED_MODULES @@ -1,4 +1,5 @@ docker +pw_alignment pw_allocator pw_analog pw_android_toolchain @@ -115,6 +116,7 @@ pw_thread_embos pw_thread_freertos pw_thread_stl pw_thread_threadx +pw_thread_zephyr pw_tls_client pw_tls_client_boringssl pw_tls_client_mbedtls @@ -180,59 +180,12 @@ load("@com_github_nanopb_nanopb//:nanopb_workspace.bzl", "nanopb_workspace") nanopb_workspace() -# Set up NodeJs rules. -# Required by: pigweed. -# Used in modules: //pw_web. -http_archive( - name = "build_bazel_rules_nodejs", - sha256 = "b32a4713b45095e9e1921a7fcb1adf584bc05959f3336e7351bcf77f015a2d7c", - urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/4.1.0/rules_nodejs-4.1.0.tar.gz"], -) - -# Get the latest LTS version of Node. -load( - "@build_bazel_rules_nodejs//:index.bzl", - "node_repositories", - "yarn_install", -) - -node_repositories(package_json = ["//:package.json"]) - -yarn_install( - name = "npm", - package_json = "//:package.json", - yarn_lock = "//:yarn.lock", -) - -# Set up web-testing rules. -# Required by: pigweed. -# Used in modules: //pw_web. -http_archive( - name = "io_bazel_rules_webtesting", - sha256 = "9bb461d5ef08e850025480bab185fd269242d4e533bca75bfb748001ceb343c3", - urls = ["https://github.com/bazelbuild/rules_webtesting/releases/download/0.3.3/rules_webtesting.tar.gz"], -) - -load("@io_bazel_rules_webtesting//web:repositories.bzl", "web_test_repositories") - -web_test_repositories() - -load( - "@io_bazel_rules_webtesting//web/versioned:browsers-0.3.2.bzl", - "browser_repositories", -) - -browser_repositories( - chromium = True, - firefox = True, -) - # Set up embedded C/C++ toolchains. # Required by: pigweed. # Used in modules: //pw_polyfill, //pw_build (all pw_cc* targets). git_repository( name = "bazel_embedded", - commit = "17c93d5fa52c4c78860b8bbd327325fba6c85555", + commit = "91dcc13ebe5df755ca2fe896ff6f7884a971d05b", remote = "https://github.com/bazelembedded/bazel-embedded.git", shallow_since = "1631751909 +0800", ) @@ -375,11 +328,6 @@ load("@rules_fuzzing//fuzzing:init.bzl", "rules_fuzzing_init") rules_fuzzing_init() -# esbuild setup -load("@build_bazel_rules_nodejs//toolchains/esbuild:esbuild_repositories.bzl", "esbuild_repositories") - -esbuild_repositories(npm_repository = "npm") # Note, npm is the default value for npm_repository - RULES_JVM_EXTERNAL_TAG = "2.8" RULES_JVM_EXTERNAL_SHA = "79c9850690d7614ecdb72d68394f994fef7534b292c4867ce5e7dec0aa7bdfad" @@ -426,3 +374,11 @@ git_repository( remote = "https://boringssl.googlesource.com/boringssl", shallow_since = "1637714942 +0000", ) + +http_archive( + name = "freertos", + build_file = "//:third_party/freertos/BUILD.bazel", + sha256 = "89af32b7568c504624f712c21fe97f7311c55fccb7ae6163cda7adde1cde7f62", + strip_prefix = "FreeRTOS-Kernel-10.5.1", + urls = ["https://github.com/FreeRTOS/FreeRTOS-Kernel/archive/refs/tags/V10.5.1.tar.gz"], +) diff --git a/bootstrap.sh b/bootstrap.sh index 429c3ac7c..a615fc501 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -63,9 +63,13 @@ export PW_PROJECT_ROOT . "$PW_ROOT/pw_env_setup/util.sh" +# Check environment properties pw_deactivate pw_eval_sourced "$_pw_sourced" "$_PW_BOOTSTRAP_PATH" -pw_check_root "$PW_ROOT" +if ! pw_check_root "$PW_ROOT"; then + return +fi + _PW_ACTUAL_ENVIRONMENT_ROOT="$(pw_get_env_root)" export _PW_ACTUAL_ENVIRONMENT_ROOT SETUP_SH="$_PW_ACTUAL_ENVIRONMENT_ROOT/activate.sh" diff --git a/docs/BUILD.gn b/docs/BUILD.gn index a45ec5085..bbfc362db 100644 --- a/docs/BUILD.gn +++ b/docs/BUILD.gn @@ -99,6 +99,7 @@ group("module_docs") { group("third_party_docs") { deps = [ "$dir_pigweed/third_party/boringssl:docs", + "$dir_pigweed/third_party/emboss:docs", "$dir_pigweed/third_party/freertos:docs", "$dir_pigweed/third_party/fuchsia:docs", "$dir_pigweed/third_party/googletest:docs", @@ -110,10 +111,27 @@ group("third_party_docs") { _doxygen_input_files = [ # All sources with doxygen comment blocks. "$dir_pw_async/public/pw_async/dispatcher.h", + "$dir_pw_async/public/pw_async/fake_dispatcher_fixture.h", "$dir_pw_async/public/pw_async/task.h", + "$dir_pw_async_basic/public/pw_async_basic/dispatcher.h", + "$dir_pw_bluetooth/public/pw_bluetooth/gatt/client.h", + "$dir_pw_bluetooth/public/pw_bluetooth/gatt/server.h", + "$dir_pw_bluetooth/public/pw_bluetooth/host.h", + "$dir_pw_bluetooth/public/pw_bluetooth/low_energy/central.h", + "$dir_pw_bluetooth/public/pw_bluetooth/low_energy/connection.h", + "$dir_pw_bluetooth/public/pw_bluetooth/low_energy/peripheral.h", "$dir_pw_chrono/public/pw_chrono/system_clock.h", "$dir_pw_chrono/public/pw_chrono/system_timer.h", + "$dir_pw_function/public/pw_function/scope_guard.h", + "$dir_pw_log_tokenized/public/pw_log_tokenized/handler.h", + "$dir_pw_log_tokenized/public/pw_log_tokenized/metadata.h", + "$dir_pw_rpc/public/pw_rpc/internal/config.h", + "$dir_pw_string/public/pw_string/format.h", + "$dir_pw_string/public/pw_string/string.h", + "$dir_pw_function/public/pw_function/function.h", + "$dir_pw_function/public/pw_function/pointer.h", "$dir_pw_string/public/pw_string/string_builder.h", + "$dir_pw_string/public/pw_string/util.h", "$dir_pw_sync/public/pw_sync/binary_semaphore.h", "$dir_pw_sync/public/pw_sync/borrow.h", "$dir_pw_sync/public/pw_sync/counting_semaphore.h", @@ -125,8 +143,8 @@ _doxygen_input_files = [ "$dir_pw_sync/public/pw_sync/timed_mutex.h", "$dir_pw_sync/public/pw_sync/timed_thread_notification.h", "$dir_pw_sync/public/pw_sync/virtual_basic_lockable.h", - "$dir_pw_rpc/public/pw_rpc/internal/config.h", "$dir_pw_tokenizer/public/pw_tokenizer/encode_args.h", + "$dir_pw_tokenizer/public/pw_tokenizer/tokenize.h", ] pw_python_action("generate_doxygen") { diff --git a/docs/Doxyfile b/docs/Doxyfile index 177de67ea..28f2b8f22 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -2387,7 +2387,10 @@ PREDEFINED = __cplusplus \ PW_UNLOCK_FUNCTION(...)= \ PW_NO_LOCK_SAFETY_ANALYSIS= \ PW_CXX_STANDARD_IS_SUPPORTED(...)=1 \ - PW_EXTERN_C_START= + PW_EXTERN_C_START= \ + PW_LOCKS_EXCLUDED(...)= \ + PW_EXCLUSIVE_LOCKS_REQUIRED(...)= \ + PW_GUARDED_BY(...)= \ # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this # tag can be used to specify a list of macro names that should be expanded. The diff --git a/docs/_static/css/pigweed.css b/docs/_static/css/pigweed.css index 3721f7fd3..bea0fee45 100644 --- a/docs/_static/css/pigweed.css +++ b/docs/_static/css/pigweed.css @@ -132,3 +132,47 @@ h4, h5, h6 { font-weight: bold; font-size: var(--font-size--normal); } + +/* Support taglines inline with page titles */ +section.with-subtitle > h1 { + display: inline; +} + +/* Restore the padding to the top of the page that was removed by making the + h1 element inline */ +section.with-subtitle { + padding-top: 1.5em; +} + +.section-subtitle { + display: inline; + font-size: larger; + font-weight: bold; +} + +/* Styling for module doc section buttons */ +ul.pw-module-section-buttons { + display: flex; + justify-content: center; + padding: 0; +} + +li.pw-module-section-button { + display: inline; + list-style-type: none; + padding: 0 4px; +} + +li.pw-module-section-button p { + display: inline; +} + +li.pw-module-section-button p a { + background-color: var(--color-section-button) !important; + border-color: var(--color-section-button) !important; +} + +li.pw-module-section-button p a:hover { + background-color: var(--color-section-button-hover) !important; + border-color: var(--color-section-button-hover) !important; +} diff --git a/docs/build_system.rst b/docs/build_system.rst index 62f9df4aa..a7a5c8e8e 100644 --- a/docs/build_system.rst +++ b/docs/build_system.rst @@ -766,9 +766,10 @@ however it is possible to override this from the command line. e.g. .. _Bazel config reference: https://docs.bazel.build/versions/main/skylark/config.html +.. _docs-build_system-bazel_configuration: -Pigweeds configuration -^^^^^^^^^^^^^^^^^^^^^^ +Pigweed's Bazel configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Pigweeds Bazel configuration API is designed to be distributed across the Pigweed repository and/or your downstream repository. If you are coming from GN's centralized configuration API it might be useful to think about diff --git a/docs/conf.py b/docs/conf.py index 39d681ca7..f99f920e5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,6 +41,7 @@ pygments_dark_style = 'pw_console.pigweed_code_style.PigweedCodeStyle' extensions = [ 'pw_docgen.sphinx.google_analytics', # Enables optional Google Analytics + 'pw_docgen.sphinx.module_metadata', 'sphinx.ext.autodoc', # Automatic documentation for Python code 'sphinx.ext.napoleon', # Parses Google-style docstrings 'sphinxarg.ext', # Automatic documentation of Python argparse @@ -151,6 +152,8 @@ html_theme_options = { 'color-highlight-on-target': '#ffffcc', # Background color emphasized code lines. 'color-code-hll-background': '#ffffcc', + 'color-section-button': '#b529aa', + 'color-section-button-hover': '#fb71fe', }, 'dark_css_variables': { 'color-sidebar-brand-text': '#fb71fe', @@ -178,6 +181,8 @@ html_theme_options = { 'color-highlight-on-target': '#ffc55140', # Background color emphasized code lines. 'color-code-hll-background': '#ffc55140', + 'color-section-button': '#fb71fe', + 'color-section-button-hover': '#b529aa', }, } diff --git a/docs/style_guide.rst b/docs/style_guide.rst index 6f7ce67c9..956ac2713 100644 --- a/docs/style_guide.rst +++ b/docs/style_guide.rst @@ -1248,7 +1248,7 @@ Doxygen comments into Sphinx. These include the following: .. admonition:: See also - `Breathe directives to use in rst files <https://breathe.readthedocs.io/en/latest/directives.html>`_ + `Breathe directives to use in RST files <https://breathe.readthedocs.io/en/latest/directives.html>`_ Example Doxygen Comment Block ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1304,46 +1304,21 @@ Start a Doxygen comment block using ``///`` (three forward slashes). Doxygen Syntax ^^^^^^^^^^^^^^ -Pigweed prefers to use rst wherever possible but there are a few Doxygen +Pigweed prefers to use RST wherever possible, but there are a few Doxygen syntatic elements that may be needed. -`Structural commands -<https://doxygen.nl/manual/docblocks.html#structuralcommands>`_ for the first -line of a Doxygen comment block. - -To group multiple things into a single comment block put them both at the -start on their own lines. For example: - -- ``@class`` to document a Class. -- ``@struct`` to document a C-struct. -- ``@union`` to document a union. -- ``@enum`` to document an enumeration type. -- ``@fn`` to document a function. -- ``@var`` to document a variable or typedef or enum value. -- ``@def`` to document a #define. -- ``@typedef`` to document a type definition. -- ``@file`` to document a file. -- ``@namespace`` to document a namespace. -- ``@package`` to document a Java package. -- ``@interface`` to document an IDL interface. +Common Doxygen commands for use within a comment block: -.. code-block:: cpp - - /// @def PW_ASSERT_EXCLUSIVE_LOCK - /// @def PW_ASSERT_SHARED_LOCK - /// - /// Documents functions that dynamically check to see if a lock is held, and - /// fail if it is not held. - -Common Doxygen commands for use within a comment block. - -- ``@rst`` To start a reStructuredText block. This is an alias for +- ``@rst`` To start a reStructuredText block. This is a custom alias for ``\verbatim embed:rst:leading-asterisk``. -- `@code <https://www.doxygen.nl/manual/commands.html#cmdcode>`_ Start a code block. +- `@code <https://www.doxygen.nl/manual/commands.html#cmdcode>`_ Start a code + block. - `@param <https://www.doxygen.nl/manual/commands.html#cmdparam>`_ Document parameters, this may be repeated. - `@pre <https://www.doxygen.nl/manual/commands.html#cmdpre>`_ Starts a paragraph where the precondition of an entity can be described. +- `@post <https://www.doxygen.nl/manual/commands.html#cmdpost>`_ Starts a + paragraph where the postcondition of an entity can be described. - `@return <https://www.doxygen.nl/manual/commands.html#cmdreturn>`_ Single paragraph to describe return value(s). - `@retval <https://www.doxygen.nl/manual/commands.html#cmdretval>`_ Document @@ -1352,6 +1327,25 @@ Common Doxygen commands for use within a comment block. admonition to the end of documentation. - `@b <https://www.doxygen.nl/manual/commands.html#cmdb>`_ To bold one word. +Doxygen provides `structural commands +<https://doxygen.nl/manual/docblocks.html#structuralcommands>`_ that associate a +comment block with a particular symbol. Example of these include ``@class``, +``@struct``, ``@def``, ``@fn``, and ``@file``. Do not use these unless +necessary, since they are redundant with the declarations themselves. + +One case where structural commands are necessary is when a single comment block +describes multiple symbols. To group multiple symbols into a single comment +block, include a structural commands for each symbol on its own line. For +example, the following comment documents two macros: + +.. code-block:: cpp + + /// @def PW_ASSERT_EXCLUSIVE_LOCK + /// @def PW_ASSERT_SHARED_LOCK + /// + /// Documents functions that dynamically check to see if a lock is held, and + /// fail if it is not held. + .. seealso:: `All Doxygen commands <https://www.doxygen.nl/manual/commands.html>`_ Cross-references diff --git a/package-lock.json b/package-lock.json index fbc143ab6..d00b93c9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,17 @@ { "name": "pigweedjs", - "version": "0.0.5", + "version": "0.0.7", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "pigweedjs", - "version": "0.0.5", + "version": "0.0.7", "license": "Apache-2.0", "dependencies": { "@protobuf-ts/protoc": "^2.7.0", "google-protobuf": "^3.17.3", + "long": "^5.2.1", "object-path": "^0.11.8", "ts-protoc-gen": "^0.15.0" }, @@ -784,6 +785,12 @@ "node": ">=6" } }, + "node_modules/@grpc/proto-loader/node_modules/long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", + "dev": true + }, "node_modules/@grpc/proto-loader/node_modules/protobufjs": { "version": "6.11.2", "dev": true, @@ -7032,9 +7039,9 @@ "license": "MIT" }, "node_modules/long": { - "version": "4.0.0", - "dev": true, - "license": "Apache-2.0" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.1.tgz", + "integrity": "sha512-GKSNGeNAtw8IryjjkhZxuKB3JzlcLTwjtiQCHKvqQet81I93kXslhDQruGI/QsddO83mcDToBVy7GqGS/zYf/A==" }, "node_modules/loose-envify": { "version": "1.4.0", @@ -10604,6 +10611,12 @@ "yargs": "^16.1.1" }, "dependencies": { + "long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", + "dev": true + }, "protobufjs": { "version": "6.11.2", "dev": true, @@ -15068,8 +15081,9 @@ "dev": true }, "long": { - "version": "4.0.0", - "dev": true + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.1.tgz", + "integrity": "sha512-GKSNGeNAtw8IryjjkhZxuKB3JzlcLTwjtiQCHKvqQet81I93kXslhDQruGI/QsddO83mcDToBVy7GqGS/zYf/A==" }, "loose-envify": { "version": "1.4.0", diff --git a/package.json b/package.json index 837da7fd5..b3eebb567 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pigweedjs", - "version": "0.0.7", + "version": "0.0.8", "description": "An open source collection of embedded-targeted libraries", "author": "The Pigweed Authors", "license": "Apache-2.0", @@ -75,6 +75,7 @@ "dependencies": { "@protobuf-ts/protoc": "^2.7.0", "google-protobuf": "^3.17.3", + "long": "^5.2.1", "object-path": "^0.11.8", "ts-protoc-gen": "^0.15.0" }, diff --git a/pigweed.json b/pigweed.json new file mode 100644 index 000000000..7a4623e5d --- /dev/null +++ b/pigweed.json @@ -0,0 +1,9 @@ +{ + "pw": { + "pw_presubmit": { + "format": { + "python_formatter": "black" + } + } + } +} diff --git a/pw_alignment/BUILD.bazel b/pw_alignment/BUILD.bazel new file mode 100644 index 000000000..298605fc9 --- /dev/null +++ b/pw_alignment/BUILD.bazel @@ -0,0 +1,26 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +load( + "//pw_build:pigweed.bzl", + "pw_cc_library", +) + +package(default_visibility = ["//visibility:public"]) + +pw_cc_library( + name = "pw_alignment", + hdrs = ["public/pw_alignment/alignment.h"], + includes = ["public"], +) diff --git a/pw_alignment/BUILD.gn b/pw_alignment/BUILD.gn new file mode 100644 index 000000000..3ea208fce --- /dev/null +++ b/pw_alignment/BUILD.gn @@ -0,0 +1,36 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import("//build_overrides/pigweed.gni") + +import("$dir_pw_build/target_types.gni") +import("$dir_pw_docgen/docs.gni") +import("$dir_pw_unit_test/test.gni") + +config("public_include_path") { + include_dirs = [ "public" ] + visibility = [ ":*" ] +} + +pw_source_set("pw_alignment") { + public = [ "public/pw_alignment/alignment.h" ] + public_configs = [ ":public_include_path" ] +} + +pw_doc_group("docs") { + sources = [ "docs.rst" ] +} + +pw_test_group("tests") { +} diff --git a/pw_alignment/docs.rst b/pw_alignment/docs.rst new file mode 100644 index 000000000..8d4958db1 --- /dev/null +++ b/pw_alignment/docs.rst @@ -0,0 +1,42 @@ +.. _module-pw_alignment: + +============ +pw_alignment +============ +This module defines an interface for ensuring natural alignment of objects. + +Avoiding Atomic Libcalls +======================== +The main motivation is to provide a portable way for +preventing the compiler from emitting libcalls to builtin atomics +functions and instead use native atomic instructions. This is especially +useful for any pigweed user that uses ``std::atomic``. + +Take for example `std::atomic<std::optional<bool>>`. Accessing the underlying object +would normally involve a call to a builtin `__atomic_*` function provided by a builtins +library. However, if the compiler can determine that the size of the object is the same +as its alignment, then it can replace a libcall to `__atomic_*` with native instructions. + +There can be certain situations where a compiler might not be able to assert this. +Depending on the implementation of `std::optional<bool>`, this object could +have a size of 2 bytes but an alignment of 1 byte which wouldn't satisfy the constraint. + +Basic usage +----------- +`pw_alignment` provides a wrapper class `pw::NaturallyAligned` for enforcing natural alignment without any +changes to how the object is used. Since this is commonly used with atomics, an +aditional class `pw::AlignedAtomic` is provided for simplifying things. + +.. code-block:: c++ + + // Passing a `std::optional<bool>` to `__atomic_exchange` might not replace the call + // with native instructions if the compiler cannot determine all instances of this object + // will happen to be aligned. + std::atomic<std::optional<bool>> maybe_nat_aligned_obj; + + // `pw::NaturallyAligned<...>` forces the object to be aligned to its size, so passing + // it to `__atomic_exchange` will result in a replacement with native instructions. + std::atomic<pw::NaturallyAligned<std::optional<bool>>> nat_aligned_obj; + + // Shorter spelling for the same as above. + std::AlignedAtomic<std::optional<bool>> also_nat_aligned_obj; diff --git a/pw_alignment/public/pw_alignment/alignment.h b/pw_alignment/public/pw_alignment/alignment.h new file mode 100644 index 000000000..e295c5d0c --- /dev/null +++ b/pw_alignment/public/pw_alignment/alignment.h @@ -0,0 +1,84 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#pragma once + +// todo-check: ignore +// TODO(fxbug.dev/120998): Once this bug is addressed, this module can likely +// be removed and we could just inline the using statements. + +#include <atomic> +#include <limits> +#include <type_traits> + +namespace pw { + +#if __cplusplus >= 202002L +using bit_ceil = std::bit_ceil; +#else +constexpr size_t countl_zero(size_t x) noexcept { + size_t size_digits = std::numeric_limits<size_t>::digits; + + if (sizeof(x) <= sizeof(unsigned int)) + return __builtin_clz(static_cast<unsigned int>(x)) - + (std::numeric_limits<unsigned int>::digits - size_digits); + + if (sizeof(x) <= sizeof(unsigned long)) + return __builtin_clzl(static_cast<unsigned long>(x)) - + (std::numeric_limits<unsigned long>::digits - size_digits); + + static_assert(sizeof(x) <= sizeof(unsigned long long)); + return __builtin_clzll(static_cast<unsigned long long>(x)) - + (std::numeric_limits<unsigned long long>::digits - size_digits); +} + +constexpr size_t bit_width(size_t x) noexcept { + return std::numeric_limits<size_t>::digits - countl_zero(x); +} + +constexpr size_t bit_ceil(size_t x) noexcept { + if (x == 0) + return 1; + return size_t{1} << bit_width(size_t{x - 1}); +} +#endif + +// The NaturallyAligned class is a wrapper class for ensuring the object is +// aligned to a power of 2 bytes greater than or equal to its size. +template <typename T> +struct [[gnu::aligned(bit_ceil(sizeof(T)))]] NaturallyAligned + : public T{NaturallyAligned() : T(){} NaturallyAligned(const T& t) : + T(t){} template <class U> + NaturallyAligned(const U& u) : T(u){} NaturallyAligned + operator=(T other){return T::operator=(other); +} // namespace pw +} +; + +// This is a convenience wrapper for ensuring the object held by std::atomic is +// naturally aligned. Ensuring the underlying objects's alignment is natural +// allows clang to replace libcalls to atomic functions +// (__atomic_load/store/exchange/etc) with native instructions when appropriate. +// +// Example usage: +// +// // Here std::optional<bool> has a size of 2 but alignment of 1, which would +// // normally lower to an __atomic_* libcall, but pw::NaturallyAligned in +// // std::atomic tells the compiler to align the object to 2 bytes, which +// // satisfies the requirements for replacing __atomic_* with instructions. +// pw::AlignedAtomic<std::optional<bool>> mute_enable{}; +// +template <typename T> +using AlignedAtomic = std::atomic<NaturallyAligned<T>>; + +} // namespace pw diff --git a/pw_arduino_build/py/pw_arduino_build/unit_test_runner.py b/pw_arduino_build/py/pw_arduino_build/unit_test_runner.py index abc4359e7..05678f751 100755 --- a/pw_arduino_build/py/pw_arduino_build/unit_test_runner.py +++ b/pw_arduino_build/py/pw_arduino_build/unit_test_runner.py @@ -25,8 +25,8 @@ import time from pathlib import Path from typing import List -import serial # type: ignore -import serial.tools.list_ports # type: ignore +import serial +import serial.tools.list_ports import pw_arduino_build.log from pw_arduino_build import teensy_detector from pw_arduino_build.file_operations import decode_file_json diff --git a/pw_arduino_build/py/setup.cfg b/pw_arduino_build/py/setup.cfg index 05f6d2a00..f59cba7f4 100644 --- a/pw_arduino_build/py/setup.cfg +++ b/pw_arduino_build/py/setup.cfg @@ -23,6 +23,7 @@ packages = find: zip_safe = False install_requires = pyserial>=3.5,<4.0 + types-pyserial>=3.5,<4.0 coloredlogs parameterized diff --git a/pw_assert/BUILD.bazel b/pw_assert/BUILD.bazel index f45ba62c5..f6d1f8616 100644 --- a/pw_assert/BUILD.bazel +++ b/pw_assert/BUILD.bazel @@ -57,6 +57,23 @@ pw_cc_library( ) pw_cc_library( + name = "libc_assert", + hdrs = [ + "libc_assert_public_overrides/assert.h", + "libc_assert_public_overrides/cassert", + "public/pw_assert/internal/libc_assert.h", + ], + includes = [ + "libc_assert_public_overrides", + "public", + ], + deps = [ + "//pw_assert", + "//pw_preprocessor", + ], +) + +pw_cc_library( name = "print_and_abort", hdrs = ["public/pw_assert/internal/print_and_abort.h"], includes = ["public"], diff --git a/pw_assert/BUILD.gn b/pw_assert/BUILD.gn index 0502d2da8..28552dc0f 100644 --- a/pw_assert/BUILD.gn +++ b/pw_assert/BUILD.gn @@ -37,6 +37,11 @@ config("assert_backend_overrides") { visibility = [ ":*" ] } +config("libc_assert_overrides") { + include_dirs = [ "libc_assert_public_overrides" ] + visibility = [ ":*" ] +} + config("print_and_abort_check_backend_overrides") { include_dirs = [ "print_and_abort_check_public_overrides" ] visibility = [ ":*" ] @@ -117,6 +122,19 @@ pw_source_set("assert_compatibility_backend") { group("assert_compatibility_backend.impl") { } +pw_source_set("libc_assert") { + public_configs = [ ":public_include_path" ] + public = [ + "libc_assert_public_overrides/assert.h", + "libc_assert_public_overrides/cassert", + "public/pw_assert/internal/libc_assert.h", + ] + public_deps = [ + ":assert", + dir_pw_preprocessor, + ] +} + pw_source_set("print_and_abort") { public_configs = [ ":public_include_path" ] public_deps = [ ":config" ] diff --git a/pw_assert/CMakeLists.txt b/pw_assert/CMakeLists.txt index 64fb40d8c..623e3efea 100644 --- a/pw_assert/CMakeLists.txt +++ b/pw_assert/CMakeLists.txt @@ -69,6 +69,19 @@ pw_add_library(pw_assert.assert_compatibility_backend INTERFACE pw_preprocessor ) +pw_add_library(pw_assert.libc_assert INTERFACE + HEADERS + libc_assert_public_overrides/assert.h + libc_assert_public_overrides/cassert + public/pw_assert/internal/libc_assert.h + PUBLIC_INCLUDES + public + libc_assert_public_overrides + PUBLIC_DEPS + pw_assert.assert + pw_preprocessor +) + pw_add_library(pw_assert.print_and_abort INTERFACE HEADERS public/pw_assert/internal/print_and_abort.h diff --git a/pw_assert/docs.rst b/pw_assert/docs.rst index 6feb29b28..f411e8184 100644 --- a/pw_assert/docs.rst +++ b/pw_assert/docs.rst @@ -769,6 +769,15 @@ Compatibility ------------- The facade is compatible with both C and C++. +--------------------------------------- +C Standard Library `assert` Replacement +--------------------------------------- +An optional replacement of the C standard Library's `assert` macro is provided +through the `libc_assert` target which fully implements replacement `assert.h` +and `cassert` headers using `PW_ASSERT`. While this is effective for porting +external code to microcontrollers, we do not advise embedded projects use the +`assert` macro unless absolutely necessary. + ---------------- Roadmap & Status ---------------- diff --git a/pw_assert/libc_assert_public_overrides/assert.h b/pw_assert/libc_assert_public_overrides/assert.h new file mode 100644 index 000000000..3d574a1f4 --- /dev/null +++ b/pw_assert/libc_assert_public_overrides/assert.h @@ -0,0 +1,16 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#pragma once + +#include "pw_assert/internal/libc_assert.h" diff --git a/pw_assert/libc_assert_public_overrides/cassert b/pw_assert/libc_assert_public_overrides/cassert new file mode 100644 index 000000000..3d574a1f4 --- /dev/null +++ b/pw_assert/libc_assert_public_overrides/cassert @@ -0,0 +1,16 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#pragma once + +#include "pw_assert/internal/libc_assert.h" diff --git a/pw_assert/public/pw_assert/internal/libc_assert.h b/pw_assert/public/pw_assert/internal/libc_assert.h new file mode 100644 index 000000000..865650ed1 --- /dev/null +++ b/pw_assert/public/pw_assert/internal/libc_assert.h @@ -0,0 +1,42 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#pragma once + +// Include these headers as C++ code, in case assert.h is included within an +// extern "C" block. +#ifdef __cplusplus +extern "C++" { +#endif // __cplusplus + +#include "pw_assert/assert.h" +#include "pw_preprocessor/util.h" + +#ifdef __cplusplus +} // extern "C++" +#endif // __cplusplus + +// Provide static_assert() on >=C11 +#if (defined(__USE_ISOC11) || \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) && \ + !defined(__cplusplus) +#define static_assert _Static_assert +#endif // C11 or newer + +// Provide assert() +#undef assert +#if defined(NDEBUG) // Required by ANSI C standard. +#define assert(condition) ((void)0) +#else +#define assert(condition) PW_ASSERT(condition) +#endif // defined(NDEBUG) diff --git a/pw_async/BUILD.bazel b/pw_async/BUILD.bazel index 7ec78637e..2ae446d6b 100644 --- a/pw_async/BUILD.bazel +++ b/pw_async/BUILD.bazel @@ -18,6 +18,7 @@ filegroup( "fake_dispatcher_test.cc", "public/pw_async/dispatcher.h", "public/pw_async/fake_dispatcher.h", + "public/pw_async/fake_dispatcher_fixture.h", "public/pw_async/internal/types.h", "public/pw_async/task.h", ], diff --git a/pw_async/BUILD.gn b/pw_async/BUILD.gn index c1e287982..483b9b5e6 100644 --- a/pw_async/BUILD.gn +++ b/pw_async/BUILD.gn @@ -16,6 +16,7 @@ import("//build_overrides/pigweed.gni") import("$dir_pw_async/async.gni") import("$dir_pw_async/backend.gni") +import("$dir_pw_async/fake_dispatcher_fixture.gni") import("$dir_pw_async/fake_dispatcher_test.gni") import("$dir_pw_build/facade.gni") import("$dir_pw_build/target_types.gni") @@ -45,6 +46,7 @@ pw_facade("task") { public_deps = [ "$dir_pw_chrono:system_clock", dir_pw_function, + dir_pw_status, ] public = [ "public/pw_async/internal/types.h", @@ -67,6 +69,14 @@ pw_facade("fake_dispatcher") { ] + pw_async_EXPERIMENTAL_MODULE_VISIBILITY } +fake_dispatcher_fixture("fake_dispatcher_fixture") { + backend = ":fake_dispatcher" + visibility = [ + ":*", + "$dir_pw_async_basic:*", + ] + pw_async_EXPERIMENTAL_MODULE_VISIBILITY +} + pw_test_group("tests") { } @@ -76,6 +86,9 @@ pw_doc_group("docs") { # Satisfy source_is_in_build_files presubmit step pw_source_set("fake_dispatcher_test") { - sources = [ "fake_dispatcher_test.cc" ] + sources = [ + "fake_dispatcher_test.cc", + "public/pw_async/fake_dispatcher_fixture.h", + ] visibility = [] } diff --git a/pw_async/docs.rst b/pw_async/docs.rst index 178066b43..e1a34e009 100644 --- a/pw_async/docs.rst +++ b/pw_async/docs.rst @@ -119,7 +119,7 @@ Next, instantiate the Dispatcher and post a task: }); // Execute `task` in 5 seconds. - dispatcher.PostDelayedTask(task, 5s); + dispatcher.PostAfter(task, 5s); // Blocks until `task` runs. work_thread.join(); @@ -133,21 +133,15 @@ the current/main thread: #include "pw_async_basic/dispatcher.h" - BasicDispatcher dispatcher; - - void interrupt_handler() { - dispatcher.PostTask([](pw::async::Context& ctx){ - // Handle interrupt - }); - } - int main() { + BasicDispatcher dispatcher; + Task task([](pw::async::Context& ctx){ printf("hello world\n"); }); // Execute `task` in 5 seconds. - dispatcher.PostDelayedTask(task, 5s); + dispatcher.PostAfter(task, 5s); dispatcher.Run(); return 0; @@ -156,23 +150,12 @@ the current/main thread: Fake Dispatcher =============== To test async code, FakeDispatcher should be dependency injected in place of -Dispatcher. Then, time should be driven in unit tests. - -.. code-block:: cpp - - TEST(Example) { - FakeDispatcher dispatcher; - - MyClass obj(&dispatcher); +Dispatcher. Then, time should be driven in unit tests using the ``Run*()`` +methods. For convenience, you can use the test fixture +FakeDispatcherFixture. - obj.ScheduleSomeTasks(); - dispatcher.RunUntilIdle(); - EXPECT_TRUE(some condition); - - obj.ScheduleTaskToRunIn30Seconds(); - dispatcher.RunFor(30s); - EXPECT_TRUE(task ran); - } +.. doxygenclass:: pw::async::test::FakeDispatcherFixture + :members: .. attention:: @@ -185,7 +168,6 @@ Dispatcher. Then, time should be driven in unit tests. Roadmap ------- - Stabilize Task cancellation API -- Create test fixture for FakeDispatcher - Utility for dynamically allocated Tasks - Bazel support - CMake support diff --git a/pw_async/fake_dispatcher_fixture.gni b/pw_async/fake_dispatcher_fixture.gni new file mode 100644 index 000000000..a58be16ac --- /dev/null +++ b/pw_async/fake_dispatcher_fixture.gni @@ -0,0 +1,38 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import("//build_overrides/pigweed.gni") +import("$dir_pw_chrono/backend.gni") +import("$dir_pw_sync/backend.gni") +import("$dir_pw_thread/backend.gni") +import("$dir_pw_unit_test/test.gni") + +# Creates a pw_source_set that provides a concrete FakeDispatcherFixture. +# +# Parameters +# +# backend (required) +# [target] The FakeDispatcher backend to use. +template("fake_dispatcher_fixture") { + assert(defined(invoker.backend)) + + pw_source_set(target_name) { + public = [ "$dir_pw_async/public/pw_async/fake_dispatcher_fixture.h" ] + public_deps = [ + "$dir_pw_unit_test", + invoker.backend, + ] + forward_variables_from(invoker, [ "visibility" ]) + } +} diff --git a/pw_async/fake_dispatcher_test.cc b/pw_async/fake_dispatcher_test.cc index 73dc090ca..3674f4cb0 100644 --- a/pw_async/fake_dispatcher_test.cc +++ b/pw_async/fake_dispatcher_test.cc @@ -14,145 +14,203 @@ #include "pw_async/fake_dispatcher.h" #include "gtest/gtest.h" -#include "pw_sync/thread_notification.h" #include "pw_thread/thread.h" #include "pw_thread_stl/options.h" +#define ASSERT_OK(status) ASSERT_EQ(OkStatus(), status) +#define ASSERT_CANCELLED(status) ASSERT_EQ(Status::Cancelled(), status) + using namespace std::chrono_literals; namespace pw::async::test { -// Lambdas can only capture one ptr worth of memory without allocating, so we -// group the data we want to share between tasks and their containing tests -// inside one struct. -struct TestPrimitives { - int count = 0; - sync::ThreadNotification notification; -}; - TEST(FakeDispatcher, PostTasks) { FakeDispatcher dispatcher; - TestPrimitives tp; - auto inc_count = [&tp]([[maybe_unused]] Context& c) { ++tp.count; }; + int count = 0; + auto inc_count = [&count]([[maybe_unused]] Context& c, Status status) { + ASSERT_OK(status); + ++count; + }; Task task(inc_count); - dispatcher.PostTask(task); + dispatcher.Post(task); Task task2(inc_count); - dispatcher.PostTask(task2); + dispatcher.Post(task2); + + Task task3(inc_count); + dispatcher.Post(task3); - Task task3([&tp]([[maybe_unused]] Context& c) { ++tp.count; }); - dispatcher.PostTask(task3); + // Should not run; RunUntilIdle() does not advance time. + Task task4([&count]([[maybe_unused]] Context& c, Status status) { + ASSERT_CANCELLED(status); + ++count; + }); + dispatcher.PostAfter(task4, 1ms); dispatcher.RunUntilIdle(); dispatcher.RequestStop(); - - ASSERT_TRUE(tp.count == 3); + dispatcher.RunUntilIdle(); + ASSERT_EQ(count, 4); } +// Lambdas can only capture one ptr worth of memory without allocating, so we +// group the data we want to share between tasks and their containing tests +// inside one struct. struct TaskPair { Task task_a; Task task_b; int count = 0; - sync::ThreadNotification notification; }; TEST(FakeDispatcher, DelayedTasks) { FakeDispatcher dispatcher; TaskPair tp; - Task task0( - [&tp]([[maybe_unused]] Context& c) { tp.count = tp.count * 10 + 4; }); - - dispatcher.PostDelayedTask(task0, 200ms); + Task task0([&tp]([[maybe_unused]] Context& c, Status status) { + ASSERT_OK(status); + tp.count = tp.count * 10 + 4; + }); + dispatcher.PostAfter(task0, 200ms); - Task task1([&tp]([[maybe_unused]] Context& c) { + Task task1([&tp]([[maybe_unused]] Context& c, Status status) { + ASSERT_OK(status); tp.count = tp.count * 10 + 1; - c.dispatcher->PostDelayedTask(tp.task_a, 50ms); - c.dispatcher->PostDelayedTask(tp.task_b, 25ms); + c.dispatcher->PostAfter(tp.task_a, 50ms); + c.dispatcher->PostAfter(tp.task_b, 25ms); }); + dispatcher.PostAfter(task1, 100ms); - dispatcher.PostDelayedTask(task1, 100ms); - - tp.task_a.set_function( - [&tp]([[maybe_unused]] Context& c) { tp.count = tp.count * 10 + 3; }); - - tp.task_b.set_function( - [&tp]([[maybe_unused]] Context& c) { tp.count = tp.count * 10 + 2; }); + tp.task_a.set_function([&tp]([[maybe_unused]] Context& c, Status status) { + ASSERT_OK(status); + tp.count = tp.count * 10 + 3; + }); + tp.task_b.set_function([&tp]([[maybe_unused]] Context& c, Status status) { + ASSERT_OK(status); + tp.count = tp.count * 10 + 2; + }); - dispatcher.RunUntilIdle(); + dispatcher.RunFor(200ms); dispatcher.RequestStop(); - - ASSERT_TRUE(tp.count == 1234); + dispatcher.RunUntilIdle(); + ASSERT_EQ(tp.count, 1234); } TEST(FakeDispatcher, CancelTasks) { FakeDispatcher dispatcher; - TestPrimitives tp; - auto inc_count = [&tp]([[maybe_unused]] Context& c) { ++tp.count; }; + auto shouldnt_run = []([[maybe_unused]] Context& c, + [[maybe_unused]] Status status) { FAIL(); }; - // This task gets canceled in the last task. - Task task0(inc_count); - dispatcher.PostDelayedTask(task0, 40ms); + TaskPair tp; + // This task gets canceled in cancel_task. + tp.task_a.set_function(shouldnt_run); + dispatcher.PostAfter(tp.task_a, 40ms); // This task gets canceled immediately. - Task task1(inc_count); - dispatcher.PostDelayedTask(task1, 10ms); + Task task1(shouldnt_run); + dispatcher.PostAfter(task1, 10ms); ASSERT_TRUE(dispatcher.Cancel(task1)); // This task cancels the first task. - Task cancel_task( - [&task0](Context& c) { ASSERT_TRUE(c.dispatcher->Cancel(task0)); }); - dispatcher.PostDelayedTask(cancel_task, 20ms); + Task cancel_task([&tp](Context& c, Status status) { + ASSERT_OK(status); + ASSERT_TRUE(c.dispatcher->Cancel(tp.task_a)); + ++tp.count; + }); + dispatcher.PostAfter(cancel_task, 20ms); - dispatcher.RunUntilIdle(); + dispatcher.RunFor(50ms); dispatcher.RequestStop(); - - ASSERT_TRUE(tp.count == 0); + dispatcher.RunUntilIdle(); + ASSERT_EQ(tp.count, 1); } // Test RequestStop() from inside task. TEST(FakeDispatcher, RequestStopInsideTask) { FakeDispatcher dispatcher; - TestPrimitives tp; - auto inc_count = [&tp]([[maybe_unused]] Context& c) { ++tp.count; }; + int count = 0; + auto cancelled_cb = [&count]([[maybe_unused]] Context& c, Status status) { + ASSERT_CANCELLED(status); + ++count; + }; // These tasks are never executed and cleaned up in RequestStop(). - Task task0(inc_count), task1(inc_count); - dispatcher.PostDelayedTask(task0, 20ms); - dispatcher.PostDelayedTask(task1, 21ms); - - Task stop_task([&tp]([[maybe_unused]] Context& c) { - ++tp.count; - c.dispatcher->RequestStop(); + Task task0(cancelled_cb), task1(cancelled_cb); + dispatcher.PostAfter(task0, 20ms); + dispatcher.PostAfter(task1, 21ms); + + Task stop_task([&count]([[maybe_unused]] Context& c, Status status) { + ASSERT_OK(status); + ++count; + static_cast<FakeDispatcher*>(c.dispatcher)->RequestStop(); + static_cast<FakeDispatcher*>(c.dispatcher)->RunUntilIdle(); }); - dispatcher.PostTask(stop_task); + dispatcher.Post(stop_task); dispatcher.RunUntilIdle(); - - ASSERT_TRUE(tp.count == 1); + ASSERT_EQ(count, 3); } TEST(FakeDispatcher, PeriodicTasks) { FakeDispatcher dispatcher; - TestPrimitives tp; - - Task periodic_task([&tp]([[maybe_unused]] Context& c) { ++tp.count; }); - dispatcher.SchedulePeriodicTask(periodic_task, 20ms, dispatcher.now() + 50ms); + int count = 0; + Task periodic_task([&count]([[maybe_unused]] Context& c, Status status) { + ASSERT_OK(status); + ++count; + }); + dispatcher.PostPeriodicAt(periodic_task, 20ms, dispatcher.now() + 50ms); // Cancel periodic task after it has run thrice, at +50ms, +70ms, and +90ms. - Task cancel_task( - [&periodic_task](Context& c) { c.dispatcher->Cancel(periodic_task); }); - dispatcher.PostDelayedTask(cancel_task, 100ms); + Task cancel_task([&periodic_task](Context& c, Status status) { + ASSERT_OK(status); + c.dispatcher->Cancel(periodic_task); + }); + dispatcher.PostAfter(cancel_task, 100ms); - dispatcher.RunUntilIdle(); + dispatcher.RunFor(300ms); dispatcher.RequestStop(); + dispatcher.RunUntilIdle(); + ASSERT_EQ(count, 3); +} + +TEST(FakeDispatcher, TasksCancelledByDispatcherDestructor) { + int count = 0; + auto inc_count = [&count]([[maybe_unused]] Context& c, Status status) { + ASSERT_CANCELLED(status); + ++count; + }; + Task task0(inc_count), task1(inc_count), task2(inc_count); + + { + FakeDispatcher dispatcher; + dispatcher.PostAfter(task0, 10s); + dispatcher.PostAfter(task1, 10s); + dispatcher.PostAfter(task2, 10s); + } + + ASSERT_EQ(count, 3); +} + +TEST(DispatcherBasic, TasksCancelledByRunFor) { + int count = 0; + auto inc_count = [&count]([[maybe_unused]] Context& c, Status status) { + ASSERT_CANCELLED(status); + ++count; + }; + Task task0(inc_count), task1(inc_count), task2(inc_count); - ASSERT_TRUE(tp.count == 3); + FakeDispatcher dispatcher; + dispatcher.PostAfter(task0, 10s); + dispatcher.PostAfter(task1, 10s); + dispatcher.PostAfter(task2, 10s); + + dispatcher.RequestStop(); + dispatcher.RunFor(5s); + ASSERT_EQ(count, 3); } } // namespace pw::async::test diff --git a/pw_async/fake_dispatcher_test.gni b/pw_async/fake_dispatcher_test.gni index c55e849f1..33a32ee33 100644 --- a/pw_async/fake_dispatcher_test.gni +++ b/pw_async/fake_dispatcher_test.gni @@ -26,6 +26,8 @@ import("$dir_pw_unit_test/test.gni") # backend (required) # [target] The FakeDispatcher backend to test. template("fake_dispatcher_test") { + assert(defined(invoker.backend)) + pw_test(target_name) { enable_if = pw_chrono_SYSTEM_CLOCK_BACKEND != "" && pw_sync_TIMED_THREAD_NOTIFICATION_BACKEND != "" && diff --git a/pw_async/public/pw_async/dispatcher.h b/pw_async/public/pw_async/dispatcher.h index 98185d95b..8faaccd67 100644 --- a/pw_async/public/pw_async/dispatcher.h +++ b/pw_async/public/pw_async/dispatcher.h @@ -20,7 +20,7 @@ namespace pw::async { class Task; /// Asynchronous Dispatcher abstract class. A default implementation is provided -/// in dispatcher_basic.h. +/// in pw_async_basic. /// /// Dispatcher implements VirtualSystemClock so the Dispatcher's time can be /// injected into other modules under test. This is useful for consistently @@ -30,46 +30,29 @@ class Dispatcher : public chrono::VirtualSystemClock { public: ~Dispatcher() override = default; - /// Stop processing tasks and break out of the task loop. - virtual void RequestStop() = 0; - /// Post caller owned |task|. - virtual void PostTask(Task& task) = 0; + virtual void Post(Task& task) = 0; /// Post caller owned |task| to be run after |delay|. - virtual void PostDelayedTask(Task& task, - chrono::SystemClock::duration delay) = 0; + virtual void PostAfter(Task& task, chrono::SystemClock::duration delay) = 0; /// Post caller owned |task| to be run at |time|. - virtual void PostTaskForTime(Task& task, - chrono::SystemClock::time_point time) = 0; + virtual void PostAt(Task& task, chrono::SystemClock::time_point time) = 0; /// Post caller owned |task| to be run immediately then rerun at a regular /// |interval|. - virtual void SchedulePeriodicTask(Task& task, - chrono::SystemClock::duration interval) = 0; - /// Post caller owned |task| to be run at |start_time| then rerun at a regular + virtual void PostPeriodic(Task& task, + chrono::SystemClock::duration interval) = 0; + /// Post caller owned |task| to be run at |time| then rerun at a regular /// |interval|. |interval| must not be zero. - virtual void SchedulePeriodicTask( - Task& task, - chrono::SystemClock::duration interval, - chrono::SystemClock::time_point start_time) = 0; + virtual void PostPeriodicAt(Task& task, + chrono::SystemClock::duration interval, + chrono::SystemClock::time_point time) = 0; /// Returns true if |task| is succesfully canceled. /// If cancelation fails, the task may be running or completed. /// Periodic tasks may be posted once more after they are canceled. virtual bool Cancel(Task& task) = 0; - - /// Execute tasks until the Dispatcher enters a state where none are queued. - virtual void RunUntilIdle() = 0; - - /// Run the Dispatcher until Now() has reached `end_time`, executing all tasks - /// that come due before then. - virtual void RunUntil(chrono::SystemClock::time_point end_time) = 0; - - /// Run the Dispatcher until `duration` has elapsed, executing all tasks that - /// come due in that period. - virtual void RunFor(chrono::SystemClock::duration duration) = 0; }; } // namespace pw::async diff --git a/pw_async/public/pw_async/fake_dispatcher.h b/pw_async/public/pw_async/fake_dispatcher.h index 04976374d..43931d032 100644 --- a/pw_async/public/pw_async/fake_dispatcher.h +++ b/pw_async/public/pw_async/fake_dispatcher.h @@ -18,71 +18,57 @@ namespace pw::async::test { -// FakeDispatcher is a facade for an implementation of Dispatcher that is used -// in unit tests. FakeDispatcher uses simulated time. RunUntil() and RunFor() -// advance time immediately, and now() returns the current simulated time. -// -// To support various Task backends, FakeDispatcher wraps a -// backend::NativeFakeDispatcher that implements standard FakeDispatcher -// behavior using backend::NativeTask objects. +/// FakeDispatcher is a facade for an implementation of Dispatcher that is used +/// in unit tests. FakeDispatcher uses simulated time. RunUntil() and RunFor() +/// advance time immediately, and now() returns the current simulated time. +/// +/// To support various Task backends, FakeDispatcher wraps a +/// backend::NativeFakeDispatcher that implements standard FakeDispatcher +/// behavior using backend::NativeTask objects. class FakeDispatcher final : public Dispatcher { public: FakeDispatcher() : native_dispatcher_(*this) {} - void RequestStop() override { native_dispatcher_.RequestStop(); } - - // Post caller owned |task|. - void PostTask(Task& task) override { native_dispatcher_.PostTask(task); } + /// Execute all runnable tasks and return without advancing simulated time. + void RunUntilIdle() { native_dispatcher_.RunUntilIdle(); } - // Post caller owned |task| to be run after |delay|. - void PostDelayedTask(Task& task, - chrono::SystemClock::duration delay) override { - native_dispatcher_.PostDelayedTask(task, delay); + /// Run the dispatcher until Now() has reached `end_time`, executing all tasks + /// that come due before then. + void RunUntil(chrono::SystemClock::time_point end_time) { + native_dispatcher_.RunUntil(end_time); } - // Post caller owned |task| to be run at |time|. - void PostTaskForTime(Task& task, - chrono::SystemClock::time_point time) override { - native_dispatcher_.PostTaskForTime(task, time); + /// Run the Dispatcher until `duration` has elapsed, executing all tasks that + /// come due in that period. + void RunFor(chrono::SystemClock::duration duration) { + native_dispatcher_.RunFor(duration); } - // Post caller owned |task| to be run immediately then rerun at a regular - // |interval|. - void SchedulePeriodicTask(Task& task, - chrono::SystemClock::duration interval) override { - native_dispatcher_.SchedulePeriodicTask(task, interval); + /// Stop processing tasks. After calling RequestStop(), the next time the + /// Dispatcher is run, all waiting Tasks will be dequeued and their + /// TaskFunctions called with a PW_STATUS_CANCELLED status. + void RequestStop() { native_dispatcher_.RequestStop(); } + + // Dispatcher overrides: + void Post(Task& task) override { native_dispatcher_.Post(task); } + void PostAfter(Task& task, chrono::SystemClock::duration delay) override { + native_dispatcher_.PostAfter(task, delay); } - // Post caller owned |task| to be run at |start_time| then rerun at a regular - // |interval|. - void SchedulePeriodicTask( - Task& task, - chrono::SystemClock::duration interval, - chrono::SystemClock::time_point start_time) override { - native_dispatcher_.SchedulePeriodicTask(task, interval, start_time); + void PostAt(Task& task, chrono::SystemClock::time_point time) override { + native_dispatcher_.PostAt(task, time); } - - // Returns true if |task| is succesfully canceled. - // If cancelation fails, the task may be running or completed. - // Periodic tasks may run once more after they are canceled. - bool Cancel(Task& task) override { return native_dispatcher_.Cancel(task); } - - // Execute tasks until the Dispatcher enters a state where none are queued. - void RunUntilIdle() override { native_dispatcher_.RunUntilIdle(); } - - // Run the Dispatcher until Now() has reached `end_time`, executing all tasks - // that come due before then. - void RunUntil(chrono::SystemClock::time_point end_time) override { - native_dispatcher_.RunUntil(end_time); + void PostPeriodic(Task& task, + chrono::SystemClock::duration interval) override { + native_dispatcher_.PostPeriodic(task, interval); } - - // Run the Dispatcher until `duration` has elapsed, executing all tasks that - // come due in that period. - void RunFor(chrono::SystemClock::duration duration) override { - native_dispatcher_.RunFor(duration); + void PostPeriodicAt(Task& task, + chrono::SystemClock::duration interval, + chrono::SystemClock::time_point start_time) override { + native_dispatcher_.PostPeriodicAt(task, interval, start_time); } + bool Cancel(Task& task) override { return native_dispatcher_.Cancel(task); } // VirtualSystemClock overrides: - chrono::SystemClock::time_point now() override { return native_dispatcher_.now(); } diff --git a/pw_async/public/pw_async/fake_dispatcher_fixture.h b/pw_async/public/pw_async/fake_dispatcher_fixture.h new file mode 100644 index 000000000..b32efeb92 --- /dev/null +++ b/pw_async/public/pw_async/fake_dispatcher_fixture.h @@ -0,0 +1,66 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#pragma once + +#include "gtest/gtest.h" +#include "pw_async/fake_dispatcher.h" + +namespace pw::async::test { + +/// Test fixture that is a simple wrapper around a FakeDispatcher. +/// +/// Example: +/// @code{.cpp} +/// using ExampleTest = pw::async::test::FakeDispatcherFixture; +/// +/// TEST_F(ExampleTest, Example) { +/// MyClass obj(dispatcher()); +/// +/// obj.ScheduleSomeTasks(); +/// RunUntilIdle(); +/// EXPECT_TRUE(some condition); +/// +/// obj.ScheduleTaskToRunIn30Seconds(); +/// RunFor(30s); +/// EXPECT_TRUE(task ran); +/// } +/// @endcode +class FakeDispatcherFixture : public ::testing::Test { + public: + /// Returns the FakeDispatcher that should be used for dependency injection. + FakeDispatcher& dispatcher() { return dispatcher_; } + + /// Returns the current fake time. + chrono::SystemClock::time_point now() { return dispatcher_.now(); } + + /// Dispatches all tasks with due times up until `now()`. + void RunUntilIdle() { dispatcher_.RunUntilIdle(); } + + /// Dispatches all tasks with due times up to `end_time`, progressively + /// advancing the fake clock. + void RunUntil(chrono::SystemClock::time_point end_time) { + dispatcher_.RunUntil(end_time); + } + + /// Dispatches all tasks with due times up to `now() + duration`, + /// progressively advancing the fake clock. + void RunFor(chrono::SystemClock::duration duration) { + dispatcher_.RunFor(duration); + } + + private: + FakeDispatcher dispatcher_; +}; + +} // namespace pw::async::test diff --git a/pw_async/public/pw_async/internal/types.h b/pw_async/public/pw_async/internal/types.h index 67b5a88c4..c8b7ea287 100644 --- a/pw_async/public/pw_async/internal/types.h +++ b/pw_async/public/pw_async/internal/types.h @@ -14,19 +14,32 @@ #pragma once #include "pw_function/function.h" +#include "pw_status/status.h" namespace pw::async { class Dispatcher; class Task; -// Task functions take a `Context` as their argument. Before executing a Task, -// the Dispatcher sets the pointer to itself and to the Task in `Context`. struct Context { Dispatcher* dispatcher; Task* task; }; -using TaskFunction = Function<void(Context&)>; +// A TaskFunction is a unit of work that is wrapped by a Task and executed on a +// Dispatcher. +// +// TaskFunctions take a `Context` as their first argument. Before executing a +// Task, the Dispatcher sets the pointer to itself and to the Task in `Context`. +// +// TaskFunctions take a `Status` as their second argument. When a Task is +// running as normal, |status| == PW_STATUS_OK. If a Task will not be able to +// run as scheduled, the Dispatcher will still invoke the TaskFunction with +// |status| == PW_STATUS_CANCELLED. This provides an opportunity to reclaim +// resources held by the Task. +// +// A Task will not run as scheduled if, for example, it is still waiting when +// the Dispatcher shuts down. +using TaskFunction = Function<void(Context&, Status)>; } // namespace pw::async diff --git a/pw_async/public/pw_async/task.h b/pw_async/public/pw_async/task.h index 657913986..c1fa8e665 100644 --- a/pw_async/public/pw_async/task.h +++ b/pw_async/public/pw_async/task.h @@ -48,7 +48,7 @@ class Task final { } /// Executes this task. - void operator()(Context& ctx) { native_type_(ctx); } + void operator()(Context& ctx, Status status) { native_type_(ctx, status); } /// Returns the inner NativeTask containing backend-specific state. Only /// Dispatcher backends or non-portable code should call these methods! diff --git a/pw_async_basic/BUILD.bazel b/pw_async_basic/BUILD.bazel index 214299ff6..8a1fb3aca 100644 --- a/pw_async_basic/BUILD.bazel +++ b/pw_async_basic/BUILD.bazel @@ -18,6 +18,7 @@ filegroup( "dispatcher.cc", "dispatcher_test.cc", "fake_dispatcher.cc", + "fake_dispatcher_fixture_test.cc", "public/pw_async_basic/dispatcher.h", "public/pw_async_basic/fake_dispatcher.h", "public/pw_async_basic/task.h", diff --git a/pw_async_basic/BUILD.gn b/pw_async_basic/BUILD.gn index e228eddef..9ccce5e85 100644 --- a/pw_async_basic/BUILD.gn +++ b/pw_async_basic/BUILD.gn @@ -15,6 +15,7 @@ import("//build_overrides/pigweed.gni") import("$dir_pw_async/async.gni") +import("$dir_pw_async/fake_dispatcher_fixture.gni") import("$dir_pw_async/fake_dispatcher_test.gni") import("$dir_pw_bloat/bloat.gni") import("$dir_pw_build/target_types.gni") @@ -81,6 +82,16 @@ fake_dispatcher_test("fake_dispatcher_test") { backend = ":fake_dispatcher" } +fake_dispatcher_fixture("fake_dispatcher_fixture") { + backend = ":fake_dispatcher" +} + +pw_test("fake_dispatcher_fixture_test") { + enable_if = pw_chrono_SYSTEM_CLOCK_BACKEND != "" + sources = [ "fake_dispatcher_fixture_test.cc" ] + deps = [ ":fake_dispatcher_fixture" ] +} + pw_source_set("dispatcher") { public_configs = [ ":public_include_path" ] public = [ "public/pw_async_basic/dispatcher.h" ] @@ -115,6 +126,7 @@ pw_test_group("tests") { tests = [ ":dispatcher_test", ":fake_dispatcher_test", + ":fake_dispatcher_fixture_test", ] } diff --git a/pw_async_basic/dispatcher.cc b/pw_async_basic/dispatcher.cc index 3ab481306..682ff4f6d 100644 --- a/pw_async_basic/dispatcher.cc +++ b/pw_async_basic/dispatcher.cc @@ -23,26 +23,40 @@ namespace pw::async { const chrono::SystemClock::duration SLEEP_DURATION = 5s; +BasicDispatcher::~BasicDispatcher() { + RequestStop(); + lock_.lock(); + DrainTaskQueue(); + lock_.unlock(); +} + void BasicDispatcher::Run() { lock_.lock(); while (!stop_requested_) { - RunLoopOnce(); + MaybeSleep(); + ExecuteDueTasks(); } + DrainTaskQueue(); lock_.unlock(); } void BasicDispatcher::RunUntilIdle() { lock_.lock(); - while (!task_queue_.empty()) { - RunLoopOnce(); + ExecuteDueTasks(); + if (stop_requested_) { + DrainTaskQueue(); } lock_.unlock(); } void BasicDispatcher::RunUntil(chrono::SystemClock::time_point end_time) { lock_.lock(); - while (end_time < now()) { - RunLoopOnce(); + while (end_time < now() && !stop_requested_) { + MaybeSleep(); + ExecuteDueTasks(); + } + if (stop_requested_) { + DrainTaskQueue(); } lock_.unlock(); } @@ -51,7 +65,7 @@ void BasicDispatcher::RunFor(chrono::SystemClock::duration duration) { RunUntil(now() + duration); } -void BasicDispatcher::RunLoopOnce() { +void BasicDispatcher::MaybeSleep() { if (task_queue_.empty() || task_queue_.front().due_time_ > now()) { // Sleep until a notification is received or until the due time of the // next task. Notifications are sent when tasks are posted or 'stop' is @@ -64,11 +78,12 @@ void BasicDispatcher::RunLoopOnce() { PW_LOG_DEBUG("no task due; waiting for signal"); timed_notification_.try_acquire_until(wake_time); lock_.lock(); - - return; } +} - while (!task_queue_.empty() && task_queue_.front().due_time_ <= now()) { +void BasicDispatcher::ExecuteDueTasks() { + while (!task_queue_.empty() && task_queue_.front().due_time_ <= now() && + !stop_requested_) { backend::NativeTask& task = task_queue_.front(); task_queue_.pop_front(); @@ -79,7 +94,7 @@ void BasicDispatcher::RunLoopOnce() { lock_.unlock(); PW_LOG_DEBUG("running task"); Context ctx{this, &task.task_}; - task(ctx); + task(ctx, OkStatus()); lock_.lock(); } } @@ -88,37 +103,49 @@ void BasicDispatcher::RequestStop() { std::lock_guard lock(lock_); PW_LOG_DEBUG("stop requested"); stop_requested_ = true; - task_queue_.clear(); timed_notification_.release(); } -void BasicDispatcher::PostTask(Task& task) { PostTaskForTime(task, now()); } +void BasicDispatcher::DrainTaskQueue() { + PW_LOG_DEBUG("draining task queue"); + while (!task_queue_.empty()) { + backend::NativeTask& task = task_queue_.front(); + task_queue_.pop_front(); + + lock_.unlock(); + PW_LOG_DEBUG("running cancelled task"); + Context ctx{this, &task.task_}; + task(ctx, Status::Cancelled()); + lock_.lock(); + } +} + +void BasicDispatcher::Post(Task& task) { PostAt(task, now()); } -void BasicDispatcher::PostDelayedTask(Task& task, - chrono::SystemClock::duration delay) { - PostTaskForTime(task, now() + delay); +void BasicDispatcher::PostAfter(Task& task, + chrono::SystemClock::duration delay) { + PostAt(task, now() + delay); } -void BasicDispatcher::PostTaskForTime(Task& task, - chrono::SystemClock::time_point time) { +void BasicDispatcher::PostAt(Task& task, chrono::SystemClock::time_point time) { lock_.lock(); PW_LOG_DEBUG("posting task"); PostTaskInternal(task.native_type(), time); lock_.unlock(); } -void BasicDispatcher::SchedulePeriodicTask( - Task& task, chrono::SystemClock::duration interval) { - SchedulePeriodicTask(task, interval, now()); +void BasicDispatcher::PostPeriodic(Task& task, + chrono::SystemClock::duration interval) { + PostPeriodicAt(task, interval, now()); } -void BasicDispatcher::SchedulePeriodicTask( +void BasicDispatcher::PostPeriodicAt( Task& task, chrono::SystemClock::duration interval, chrono::SystemClock::time_point start_time) { PW_DCHECK(interval != chrono::SystemClock::duration::zero()); task.native_type().set_interval(interval); - PostTaskForTime(task, start_time); + PostAt(task, start_time); } bool BasicDispatcher::Cancel(Task& task) { @@ -126,7 +153,6 @@ bool BasicDispatcher::Cancel(Task& task) { return task_queue_.remove(task.native_type()); } -// Ensure lock_ is held when invoking this function. void BasicDispatcher::PostTaskInternal( backend::NativeTask& task, chrono::SystemClock::time_point time_due) { task.due_time_ = time_due; diff --git a/pw_async_basic/dispatcher_test.cc b/pw_async_basic/dispatcher_test.cc index 2c47d4e6d..3f72bbb4f 100644 --- a/pw_async_basic/dispatcher_test.cc +++ b/pw_async_basic/dispatcher_test.cc @@ -19,6 +19,9 @@ #include "pw_thread/thread.h" #include "pw_thread_stl/options.h" +#define ASSERT_OK(status) ASSERT_EQ(OkStatus(), status) +#define ASSERT_CANCELLED(status) ASSERT_EQ(Status::Cancelled(), status) + using namespace std::chrono_literals; namespace pw::async { @@ -36,25 +39,28 @@ TEST(DispatcherBasic, PostTasks) { thread::Thread work_thread(thread::stl::Options(), dispatcher); TestPrimitives tp; - auto inc_count = [&tp]([[maybe_unused]] Context& c) { ++tp.count; }; + auto inc_count = [&tp]([[maybe_unused]] Context& c, Status status) { + ASSERT_OK(status); + ++tp.count; + }; Task task(inc_count); - dispatcher.PostTask(task); + dispatcher.Post(task); Task task2(inc_count); - dispatcher.PostTask(task2); + dispatcher.Post(task2); - Task task3([&tp]([[maybe_unused]] Context& c) { + Task task3([&tp]([[maybe_unused]] Context& c, Status status) { + ASSERT_OK(status); ++tp.count; tp.notification.release(); }); - dispatcher.PostTask(task3); + dispatcher.Post(task3); tp.notification.acquire(); dispatcher.RequestStop(); work_thread.join(); - - ASSERT_TRUE(tp.count == 3); + ASSERT_EQ(tp.count, 3); } struct TaskPair { @@ -68,32 +74,26 @@ TEST(DispatcherBasic, ChainedTasks) { BasicDispatcher dispatcher; thread::Thread work_thread(thread::stl::Options(), dispatcher); - TaskPair tp; - - Task task0([&tp](Context& c) { - ++tp.count; - - c.dispatcher->PostTask(tp.task_a); + sync::ThreadNotification notification; + Task task1([¬ification]([[maybe_unused]] Context& c, Status status) { + ASSERT_OK(status); + notification.release(); }); - tp.task_a.set_function([&tp](Context& c) { - ++tp.count; - - c.dispatcher->PostTask(tp.task_b); + Task task2([&task1](Context& c, Status status) { + ASSERT_OK(status); + c.dispatcher->Post(task1); }); - tp.task_b.set_function([&tp]([[maybe_unused]] Context& c) { - ++tp.count; - tp.notification.release(); + Task task3([&task2](Context& c, Status status) { + ASSERT_OK(status); + c.dispatcher->Post(task2); }); + dispatcher.Post(task3); - dispatcher.PostTask(task0); - - tp.notification.acquire(); + notification.acquire(); dispatcher.RequestStop(); work_thread.join(); - - ASSERT_TRUE(tp.count == 3); } // Test RequestStop() from inside task. @@ -101,25 +101,100 @@ TEST(DispatcherBasic, RequestStopInsideTask) { BasicDispatcher dispatcher; thread::Thread work_thread(thread::stl::Options(), dispatcher); - TestPrimitives tp; - auto inc_count = [&tp]([[maybe_unused]] Context& c) { ++tp.count; }; + int count = 0; + auto inc_count = [&count]([[maybe_unused]] Context& c, Status status) { + ASSERT_CANCELLED(status); + ++count; + }; // These tasks are never executed and cleaned up in RequestStop(). Task task0(inc_count), task1(inc_count); - dispatcher.PostDelayedTask(task0, 20ms); - dispatcher.PostDelayedTask(task1, 21ms); + dispatcher.PostAfter(task0, 20ms); + dispatcher.PostAfter(task1, 21ms); - Task stop_task([&tp]([[maybe_unused]] Context& c) { - ++tp.count; - c.dispatcher->RequestStop(); - tp.notification.release(); + Task stop_task([&count]([[maybe_unused]] Context& c, Status status) { + ASSERT_OK(status); + ++count; + static_cast<BasicDispatcher*>(c.dispatcher)->RequestStop(); }); - dispatcher.PostTask(stop_task); + dispatcher.Post(stop_task); - tp.notification.acquire(); work_thread.join(); + ASSERT_EQ(count, 3); +} + +TEST(DispatcherBasic, TasksCancelledByRequestStopInDifferentThread) { + BasicDispatcher dispatcher; + thread::Thread work_thread(thread::stl::Options(), dispatcher); - ASSERT_TRUE(tp.count == 1); + int count = 0; + auto inc_count = [&count]([[maybe_unused]] Context& c, Status status) { + ASSERT_CANCELLED(status); + ++count; + }; + + Task task0(inc_count), task1(inc_count), task2(inc_count); + dispatcher.PostAfter(task0, 10s); + dispatcher.PostAfter(task1, 10s); + dispatcher.PostAfter(task2, 10s); + + dispatcher.RequestStop(); + work_thread.join(); + ASSERT_EQ(count, 3); +} + +TEST(DispatcherBasic, TasksCancelledByDispatcherDestructor) { + int count = 0; + auto inc_count = [&count]([[maybe_unused]] Context& c, Status status) { + ASSERT_CANCELLED(status); + ++count; + }; + Task task0(inc_count), task1(inc_count), task2(inc_count); + + { + BasicDispatcher dispatcher; + dispatcher.PostAfter(task0, 10s); + dispatcher.PostAfter(task1, 10s); + dispatcher.PostAfter(task2, 10s); + } + + ASSERT_EQ(count, 3); +} + +TEST(DispatcherBasic, TasksCancelledByRunUntilIdle) { + int count = 0; + auto inc_count = [&count]([[maybe_unused]] Context& c, Status status) { + ASSERT_CANCELLED(status); + ++count; + }; + Task task0(inc_count), task1(inc_count), task2(inc_count); + + BasicDispatcher dispatcher; + dispatcher.PostAfter(task0, 10s); + dispatcher.PostAfter(task1, 10s); + dispatcher.PostAfter(task2, 10s); + + dispatcher.RequestStop(); + dispatcher.RunUntilIdle(); + ASSERT_EQ(count, 3); +} + +TEST(DispatcherBasic, TasksCancelledByRunFor) { + int count = 0; + auto inc_count = [&count]([[maybe_unused]] Context& c, Status status) { + ASSERT_CANCELLED(status); + ++count; + }; + Task task0(inc_count), task1(inc_count), task2(inc_count); + + BasicDispatcher dispatcher; + dispatcher.PostAfter(task0, 10s); + dispatcher.PostAfter(task1, 10s); + dispatcher.PostAfter(task2, 10s); + + dispatcher.RequestStop(); + dispatcher.RunFor(5s); + ASSERT_EQ(count, 3); } } // namespace pw::async diff --git a/pw_async_basic/docs.rst b/pw_async_basic/docs.rst index fcf26a9ce..0363a2683 100644 --- a/pw_async_basic/docs.rst +++ b/pw_async_basic/docs.rst @@ -7,9 +7,15 @@ pw_async_basic This module includes basic implementations of pw_async's Dispatcher and FakeDispatcher. -Usage -===== +--- +API +--- +.. doxygenclass:: pw::async::BasicDispatcher + :members: +----- +Usage +----- First, set the following GN variables: .. code-block:: @@ -36,7 +42,7 @@ Next, construct and use a ``BasicDispatcher``. #include "pw_async_basic/dispatcher.h" void DelayedPrint(pw::async::Dispatcher& dispatcher) { - dispatcher.PostDelayedTask([](auto&){ + dispatcher.PostAfter([](auto&){ printf("hello world\n"); }, 5s); } @@ -48,8 +54,7 @@ Next, construct and use a ``BasicDispatcher``. return 0; } - +----------- Size Report -=========== - +----------- .. include:: docs_size_report diff --git a/pw_async_basic/fake_dispatcher.cc b/pw_async_basic/fake_dispatcher.cc index ac999fd2e..cbba89e98 100644 --- a/pw_async_basic/fake_dispatcher.cc +++ b/pw_async_basic/fake_dispatcher.cc @@ -24,21 +24,28 @@ namespace pw::async::test::backend { NativeFakeDispatcher::NativeFakeDispatcher(Dispatcher& dispatcher) : dispatcher_(dispatcher) {} -NativeFakeDispatcher::~NativeFakeDispatcher() { RequestStop(); } +NativeFakeDispatcher::~NativeFakeDispatcher() { + RequestStop(); + DrainTaskQueue(); +} void NativeFakeDispatcher::RunUntilIdle() { - while (!task_queue_.empty()) { - // Only advance to the due time of the next task because new tasks can be - // scheduled in the next task. - now_ = task_queue_.front().due_time(); - RunLoopOnce(); + ExecuteDueTasks(); + if (stop_requested_) { + DrainTaskQueue(); } } void NativeFakeDispatcher::RunUntil(chrono::SystemClock::time_point end_time) { - while (!task_queue_.empty() && task_queue_.front().due_time() <= end_time) { + while (!task_queue_.empty() && task_queue_.front().due_time() <= end_time && + !stop_requested_) { now_ = task_queue_.front().due_time(); - RunLoopOnce(); + ExecuteDueTasks(); + } + + if (stop_requested_) { + DrainTaskQueue(); + return; } if (now_ < end_time) { @@ -50,8 +57,9 @@ void NativeFakeDispatcher::RunFor(chrono::SystemClock::duration duration) { RunUntil(now() + duration); } -void NativeFakeDispatcher::RunLoopOnce() { - while (!task_queue_.empty() && task_queue_.front().due_time() <= now()) { +void NativeFakeDispatcher::ExecuteDueTasks() { + while (!task_queue_.empty() && task_queue_.front().due_time() <= now() && + !stop_requested_) { ::pw::async::backend::NativeTask& task = task_queue_.front(); task_queue_.pop_front(); @@ -60,41 +68,50 @@ void NativeFakeDispatcher::RunLoopOnce() { } Context ctx{&dispatcher_, &task.task_}; - task(ctx); + task(ctx, OkStatus()); } } void NativeFakeDispatcher::RequestStop() { PW_LOG_DEBUG("stop requested"); - task_queue_.clear(); + stop_requested_ = true; } -void NativeFakeDispatcher::PostTask(Task& task) { - PostTaskForTime(task, now()); +void NativeFakeDispatcher::DrainTaskQueue() { + while (!task_queue_.empty()) { + ::pw::async::backend::NativeTask& task = task_queue_.front(); + task_queue_.pop_front(); + + PW_LOG_DEBUG("running cancelled task"); + Context ctx{&dispatcher_, &task.task_}; + task(ctx, Status::Cancelled()); + } } -void NativeFakeDispatcher::PostDelayedTask( - Task& task, chrono::SystemClock::duration delay) { - PostTaskForTime(task, now() + delay); +void NativeFakeDispatcher::Post(Task& task) { PostAt(task, now()); } + +void NativeFakeDispatcher::PostAfter(Task& task, + chrono::SystemClock::duration delay) { + PostAt(task, now() + delay); } -void NativeFakeDispatcher::PostTaskForTime( - Task& task, chrono::SystemClock::time_point time) { +void NativeFakeDispatcher::PostAt(Task& task, + chrono::SystemClock::time_point time) { PW_LOG_DEBUG("posting task"); PostTaskInternal(task.native_type(), time); } -void NativeFakeDispatcher::SchedulePeriodicTask( +void NativeFakeDispatcher::PostPeriodic( Task& task, chrono::SystemClock::duration interval) { - SchedulePeriodicTask(task, interval, now()); + PostPeriodicAt(task, interval, now()); } -void NativeFakeDispatcher::SchedulePeriodicTask( +void NativeFakeDispatcher::PostPeriodicAt( Task& task, chrono::SystemClock::duration interval, chrono::SystemClock::time_point start_time) { task.native_type().set_interval(interval); - PostTaskForTime(task, start_time); + PostAt(task, start_time); } bool NativeFakeDispatcher::Cancel(Task& task) { diff --git a/pw_async_basic/fake_dispatcher_fixture_test.cc b/pw_async_basic/fake_dispatcher_fixture_test.cc new file mode 100644 index 000000000..5262c3438 --- /dev/null +++ b/pw_async_basic/fake_dispatcher_fixture_test.cc @@ -0,0 +1,36 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#include "pw_async/fake_dispatcher_fixture.h" + +#include "gtest/gtest.h" + +namespace pw::async { +namespace { + +using FakeDispatcherFixture = test::FakeDispatcherFixture; + +TEST_F(FakeDispatcherFixture, PostTasks) { + int count = 0; + auto inc_count = [&count](Context& /*c*/, Status /*status*/) { ++count; }; + + Task task(inc_count); + dispatcher().Post(task); + + ASSERT_EQ(count, 0); + RunUntilIdle(); + ASSERT_EQ(count, 1); +} + +} // namespace +} // namespace pw::async diff --git a/pw_async_basic/public/pw_async_basic/dispatcher.h b/pw_async_basic/public/pw_async_basic/dispatcher.h index 2e31cde3c..7050d6622 100644 --- a/pw_async_basic/public/pw_async_basic/dispatcher.h +++ b/pw_async_basic/public/pw_async_basic/dispatcher.h @@ -22,55 +22,73 @@ namespace pw::async { +/// BasicDispatcher is a generic implementation of Dispatcher. class BasicDispatcher final : public Dispatcher, public thread::ThreadCore { public: - explicit BasicDispatcher() : stop_requested_(false) {} - ~BasicDispatcher() override { RequestStop(); } + explicit BasicDispatcher() = default; + ~BasicDispatcher() override; + + /// Execute all runnable tasks and return without waiting. + void RunUntilIdle(); + + /// Run the dispatcher until Now() has reached `end_time`, executing all tasks + /// that come due before then. + void RunUntil(chrono::SystemClock::time_point end_time); + + /// Run the dispatcher until `duration` has elapsed, executing all tasks that + /// come due in that period. + void RunFor(chrono::SystemClock::duration duration); + + /// Stop processing tasks. If the dispatcher is serving a task loop, break out + /// of the loop, dequeue all waiting tasks, and call their TaskFunctions with + /// a PW_STATUS_CANCELLED status. If no task loop is being served, execute the + /// dequeueing procedure the next time the Dispatcher is run. + void RequestStop() PW_LOCKS_EXCLUDED(lock_); + + // ThreadCore overrides: + + /// Run the dispatcher until RequestStop() is called. Overrides + /// ThreadCore::Run() so that BasicDispatcher is compatible with + /// pw::thread::Thread. + void Run() override PW_LOCKS_EXCLUDED(lock_); // Dispatcher overrides: - void RequestStop() override PW_LOCKS_EXCLUDED(lock_); - void PostTask(Task& task) override; - void PostDelayedTask(Task& task, - chrono::SystemClock::duration delay) override; - void PostTaskForTime(Task& task, - chrono::SystemClock::time_point time) override; - void SchedulePeriodicTask(Task& task, - chrono::SystemClock::duration interval) override; - void SchedulePeriodicTask( - Task& task, - chrono::SystemClock::duration interval, - chrono::SystemClock::time_point start_time) override; + void Post(Task& task) override; + void PostAfter(Task& task, chrono::SystemClock::duration delay) override; + void PostAt(Task& task, chrono::SystemClock::time_point time) override; + void PostPeriodic(Task& task, + chrono::SystemClock::duration interval) override; + void PostPeriodicAt(Task& task, + chrono::SystemClock::duration interval, + chrono::SystemClock::time_point start_time) override; bool Cancel(Task& task) override PW_LOCKS_EXCLUDED(lock_); - void RunUntilIdle() override; - void RunUntil(chrono::SystemClock::time_point end_time) override; - void RunFor(chrono::SystemClock::duration duration) override; // VirtualSystemClock overrides: chrono::SystemClock::time_point now() override { return chrono::SystemClock::now(); } - // ThreadCore overrides: - void Run() override PW_LOCKS_EXCLUDED(lock_); - private: // Insert |task| into task_queue_ maintaining its min-heap property, keyed by - // |time_due|. Must be holding lock_ when calling this function. + // |time_due|. void PostTaskInternal(backend::NativeTask& task, chrono::SystemClock::time_point time_due) PW_EXCLUSIVE_LOCKS_REQUIRED(lock_); - // If no tasks are due, sleeps until a notification is received or until the - // due time of the next task. - // - // If at least one task is due, dequeues and runs each task that is due. - // - // Must be holding lock_ when calling this function. - void RunLoopOnce() PW_EXCLUSIVE_LOCKS_REQUIRED(lock_); + // If no tasks are due, sleep until a notification is received, the next task + // comes due, or a timeout elapses; whichever occurs first. + void MaybeSleep() PW_EXCLUSIVE_LOCKS_REQUIRED(lock_); + + // Dequeue and run each task that is due. + void ExecuteDueTasks() PW_EXCLUSIVE_LOCKS_REQUIRED(lock_); + + // Dequeue each task and call each TaskFunction with a PW_STATUS_CANCELLED + // status. + void DrainTaskQueue() PW_EXCLUSIVE_LOCKS_REQUIRED(lock_); sync::InterruptSpinLock lock_; sync::TimedThreadNotification timed_notification_; - bool stop_requested_ PW_GUARDED_BY(lock_); + bool stop_requested_ PW_GUARDED_BY(lock_) = false; // A priority queue of scheduled Tasks sorted by earliest due times first. IntrusiveList<backend::NativeTask> task_queue_ PW_GUARDED_BY(lock_); }; diff --git a/pw_async_basic/public/pw_async_basic/fake_dispatcher.h b/pw_async_basic/public/pw_async_basic/fake_dispatcher.h index 49938fb78..96fd7aae8 100644 --- a/pw_async_basic/public/pw_async_basic/fake_dispatcher.h +++ b/pw_async_basic/public/pw_async_basic/fake_dispatcher.h @@ -26,16 +26,16 @@ class NativeFakeDispatcher final { void RequestStop(); - void PostTask(Task& task); + void Post(Task& task); - void PostDelayedTask(Task& task, chrono::SystemClock::duration delay); + void PostAfter(Task& task, chrono::SystemClock::duration delay); - void PostTaskForTime(Task& task, chrono::SystemClock::time_point time); + void PostAt(Task& task, chrono::SystemClock::time_point time); - void SchedulePeriodicTask(Task& task, chrono::SystemClock::duration interval); - void SchedulePeriodicTask(Task& task, - chrono::SystemClock::duration interval, - chrono::SystemClock::time_point start_time); + void PostPeriodic(Task& task, chrono::SystemClock::duration interval); + void PostPeriodicAt(Task& task, + chrono::SystemClock::duration interval, + chrono::SystemClock::time_point start_time); bool Cancel(Task& task); @@ -53,10 +53,15 @@ class NativeFakeDispatcher final { void PostTaskInternal(::pw::async::backend::NativeTask& task, chrono::SystemClock::time_point time_due); - // Executes all pending tasks with a due time <= now(). - void RunLoopOnce(); + // Dequeue and run each task that is due. + void ExecuteDueTasks(); + + // Dequeue each task and run each TaskFunction with a PW_STATUS_CANCELLED + // status. + void DrainTaskQueue(); Dispatcher& dispatcher_; + bool stop_requested_ = false; // A priority queue of scheduled tasks sorted by earliest due times first. IntrusiveList<::pw::async::backend::NativeTask> task_queue_; diff --git a/pw_async_basic/public/pw_async_basic/task.h b/pw_async_basic/public/pw_async_basic/task.h index 1d7247f66..d83ea34ab 100644 --- a/pw_async_basic/public/pw_async_basic/task.h +++ b/pw_async_basic/public/pw_async_basic/task.h @@ -36,7 +36,7 @@ class NativeTask final : public IntrusiveList<NativeTask>::Item { NativeTask(::pw::async::Task& task) : task_(task) {} explicit NativeTask(::pw::async::Task& task, TaskFunction&& f) : func_(std::move(f)), task_(task) {} - void operator()(Context& ctx) { func_(ctx); } + void operator()(Context& ctx, Status status) { func_(ctx, status); } void set_function(TaskFunction&& f) { func_ = std::move(f); } pw::chrono::SystemClock::time_point due_time() const { return due_time_; } diff --git a/pw_async_basic/size_report/post_1_task.cc b/pw_async_basic/size_report/post_1_task.cc index f2e4da470..ef6065b1c 100644 --- a/pw_async_basic/size_report/post_1_task.cc +++ b/pw_async_basic/size_report/post_1_task.cc @@ -21,7 +21,7 @@ int main() { pw::async::BasicDispatcher dispatcher; pw::async::Task task( [](pw::async::Context& /*ctx*/) { printf("hello world\n"); }); - dispatcher.PostTask(task); + dispatcher.Post(task); dispatcher.Run(); return 0; } diff --git a/pw_bloat/py/pw_bloat/label_output.py b/pw_bloat/py/pw_bloat/label_output.py index cd3ad32d6..938cc90e2 100644 --- a/pw_bloat/py/pw_bloat/label_output.py +++ b/pw_bloat/py/pw_bloat/label_output.py @@ -173,7 +173,7 @@ class BloatTableOutput: if old_labels is None: return new_labels diff_list = [] - for (new_lb, old_lb) in zip(new_labels, old_labels): + for new_lb, old_lb in zip(new_labels, old_labels): if (new_lb.name == old_lb.name) and (new_lb.size == old_lb.size): diff_list.append(self._LabelContent('', 0, '')) else: @@ -248,7 +248,6 @@ class BloatTableOutput: if (len(self._ascii_table_rows) > 5) and ( self._ascii_table_rows[-1][0] != '+' ): - self._ascii_table_rows.append( self._row_divider( len(self._col_names), self._cs.H.value diff --git a/pw_bluetooth/BUILD.gn b/pw_bluetooth/BUILD.gn index 8e50943c7..7f7ea2c0d 100644 --- a/pw_bluetooth/BUILD.gn +++ b/pw_bluetooth/BUILD.gn @@ -91,6 +91,11 @@ if (dir_pw_third_party_emboss != "") { imports = [ "public/pw_bluetooth/hci.emb" ] deps = [ ":emboss_hci" ] } +} else { + group("emboss_hci") { + } + group("emboss_vendor") { + } } pw_test_group("tests") { diff --git a/pw_bluetooth/docs.rst b/pw_bluetooth/docs.rst index 62bad337b..428f62513 100644 --- a/pw_bluetooth/docs.rst +++ b/pw_bluetooth/docs.rst @@ -17,6 +17,76 @@ The headers in `public/pw_bluetooth` constitute a Bluetooth Host API. `host.h` is the entry point from which all other APIs are exposed. Currently, only Low Energy APIs exist. +Host +==== +.. doxygenclass:: pw::bluetooth::Host + :members: + +low_energy::Central +=================== +.. doxygenclass:: pw::bluetooth::low_energy::Central + :members: + +low_energy::Peripheral +====================== +.. doxygenclass:: pw::bluetooth::low_energy::Peripheral + :members: + +low_energy::AdvertisedPeripheral +================================ +.. doxygenclass:: pw::bluetooth::low_energy::AdvertisedPeripheral + :members: + +low_energy::Connection +====================== +.. doxygenclass:: pw::bluetooth::low_energy::Connection + :members: + +low_energy::ConnectionOptions +============================= +.. doxygenstruct:: pw::bluetooth::low_energy::ConnectionOptions + :members: + +low_energy::RequestedConnectionParameters +========================================= +.. doxygenstruct:: pw::bluetooth::low_energy::RequestedConnectionParameters + :members: + +low_energy::ConnectionParameters +================================ +.. doxygenstruct:: pw::bluetooth::low_energy::ConnectionParameters + :members: + +gatt::Server +============ +.. doxygenclass:: pw::bluetooth::gatt::Server + :members: + +gatt::LocalServiceInfo +====================== +.. doxygenstruct:: pw::bluetooth::gatt::LocalServiceInfo + :members: + +gatt::LocalService +================== +.. doxygenclass:: pw::bluetooth::gatt::LocalService + :members: + +gatt::LocalServiceDelegate +========================== +.. doxygenclass:: pw::bluetooth::gatt::LocalServiceDelegate + :members: + +gatt::Client +============ +.. doxygenclass:: pw::bluetooth::gatt::Client + :members: + +gatt::RemoteService +=================== +.. doxygenclass:: pw::bluetooth::gatt::RemoteService + :members: + Callbacks ========= This module contains callback-heavy APIs. Callbacks must not call back into the diff --git a/pw_bluetooth/public/pw_bluetooth/gatt/client.h b/pw_bluetooth/public/pw_bluetooth/gatt/client.h index a8346f8ed..e12de22be 100644 --- a/pw_bluetooth/public/pw_bluetooth/gatt/client.h +++ b/pw_bluetooth/public/pw_bluetooth/gatt/client.h @@ -27,86 +27,86 @@ namespace pw::bluetooth::gatt { -// Represents a GATT service on a remote GATT server. -// Clients should call `SetErrorCallback` before using in order to handle fatal -// errors. +/// Represents a GATT service on a remote GATT server. +/// Clients should call `SetErrorCallback` before using in order to handle fatal +/// errors. class RemoteService { public: enum class RemoteServiceError { - // The service has been modified or removed. + /// The service has been modified or removed. kServiceRemoved = 0, - // The peer serving this service has disconnected. + /// The peer serving this service has disconnected. kPeerDisconnected = 1, }; - // Wrapper around a possible truncated value received from the server. + /// Wrapper around a possible truncated value received from the server. struct ReadValue { - // Characteristic or descriptor handle. + /// Characteristic or descriptor handle. Handle handle; - // The value of the characteristic or descriptor. + /// The value of the characteristic or descriptor. Vector<std::byte> value; - // True if `value` might be truncated (the buffer was completely filled by - // the server and the read was a short read). `ReadCharacteristic` or - // `ReadDescriptor` should be used to read the complete value. + /// True if `value` might be truncated (the buffer was completely filled by + /// the server and the read was a short read). `ReadCharacteristic` or + /// `ReadDescriptor` should be used to read the complete value. bool maybe_truncated; }; - // A result returned by `ReadByType`. + /// A result returned by `ReadByType`. struct ReadByTypeResult { - // Characteristic or descriptor handle. + /// Characteristic or descriptor handle. Handle handle; - // The value of the characteristic or descriptor, if it was read - // successfully, or an error explaining why the value could not be read. + /// The value of the characteristic or descriptor, if it was read + /// successfully, or an error explaining why the value could not be read. Result<Error, ReadValue> result; }; - // Represents the supported options to read a long characteristic or - // descriptor value from a server. Long values are those that may not fit in a - // single message (longer than 22 bytes). + /// Represents the supported options to read a long characteristic or + /// descriptor value from a server. Long values are those that may not fit in + /// a single message (longer than 22 bytes). struct LongReadOptions { - // The byte to start the read at. Must be less than the length of the - // value. + /// The byte to start the read at. Must be less than the length of the + /// value. uint16_t offset = 0; - // The maximum number of bytes to read. + /// The maximum number of bytes to read. uint16_t max_bytes = kMaxValueLength; }; - // Represents the supported write modes for writing characteristics & - // descriptors to the server. + /// Represents the supported write modes for writing characteristics & + /// descriptors to the server. enum class WriteMode : uint8_t { - // Wait for a response from the server before returning but do not verify - // the echo response. Supported for both characteristics and descriptors. + /// Wait for a response from the server before returning but do not verify + /// the echo response. Supported for both characteristics and descriptors. kDefault = 0, - // Every value blob is verified against an echo response from the server. - // The procedure is aborted if a value blob has not been reliably delivered - // to the peer. Only supported for characteristics. + /// Every value blob is verified against an echo response from the server. + /// The procedure is aborted if a value blob has not been reliably delivered + /// to the peer. Only supported for characteristics. kReliable = 1, - // Delivery will not be confirmed before returning. Writing without a - // response is only supported for short characteristics with the - // `WRITE_WITHOUT_RESPONSE` property. The value must fit into a single - // message. It is guaranteed that at least 20 bytes will fit into a single - // message. If the value does not fit, a `kFailure` error will be produced. - // The value will be written at offset 0. Only supported for - // characteristics. + /// Delivery will not be confirmed before returning. Writing without a + /// response is only supported for short characteristics with the + /// `WRITE_WITHOUT_RESPONSE` property. The value must fit into a single + /// message. It is guaranteed that at least 20 bytes will fit into a single + /// message. If the value does not fit, a `kFailure` error will be produced. + /// The value will be written at offset 0. Only supported for + /// characteristics. kWithoutResponse = 2, }; - // Represents the supported options to write a characteristic/descriptor value - // to a server. + /// Represents the supported options to write a characteristic/descriptor + /// value to a server. struct WriteOptions { - // The mode of the write operation. For descriptors, only - // `WriteMode::kDefault` is supported + /// The mode of the write operation. For descriptors, only + /// `WriteMode::kDefault` is supported WriteMode mode = WriteMode::kDefault; - // Request a write starting at the byte indicated. - // Must be 0 if `mode` is `WriteMode.kWithoutResponse`. + /// Request a write starting at the byte indicated. + /// Must be 0 if `mode` is `WriteMode.kWithoutResponse`. uint16_t offset = 0; }; @@ -114,181 +114,193 @@ class RemoteService { using ReadCallback = Function<void(Result<ReadValue>)>; using NotificationCallback = Function<void(ReadValue)>; - // Set a callback that will be called when there is an error with this - // RemoteService, after which this RemoteService will be invalid. + /// Set a callback that will be called when there is an error with this + /// RemoteService, after which this RemoteService will be invalid. void SetErrorCallback(Function<void(RemoteServiceError)>&& error_callback); - // Calls `characteristic_callback` with the characteristics and descriptors in - // this service. + /// Calls `characteristic_callback` with the characteristics and descriptors + /// in this service. void DiscoverCharacteristics( Function<void(Characteristic)>&& characteristic_callback); - // Reads characteristics and descriptors with the specified type. This method - // is useful for reading values before discovery has completed, thereby - // reducing latency. - // `uuid` - The UUID of the characteristics/descriptors to read. - // `result_callback` - Results are returned via this callback. Results may be - // empty if no matching values are read. If reading a value results in a - // permission error, the handle and error will be included. - // - // This may fail with the following errors: - // kInvalidParameters: if `uuid` refers to an internally reserved descriptor - // type (e.g. the Client Characteristic Configuration descriptor). - // kTooManyResults: More results were read than can fit - // in a Vector. Consider reading characteristics/descriptors individually - // after performing discovery. - // kFailure: The server returned an error not specific to a single result. + /// Reads characteristics and descriptors with the specified type. This method + /// is useful for reading values before discovery has completed, thereby + /// reducing latency. + /// @param uuid The UUID of the characteristics/descriptors to read. + /// @param result_callback Results are returned via this callback. Results may + /// be empty if no matching values are read. If reading a value results in a + /// permission error, the handle and error will be included. + /// + /// This may fail with the following errors: + /// - kInvalidParameters: if `uuid` refers to an internally reserved + /// descriptor type (e.g. the Client Characteristic Configuration descriptor). + /// - kTooManyResults: More results were read than can fit in a Vector. + /// Consider reading characteristics/descriptors individually after performing + /// discovery. + /// - kFailure: The server returned an error not specific to a single result. void ReadByType(Uuid uuid, ReadByTypeCallback&& result_callback); - // Reads the value of a characteristic. - // `handle` - The handle of the characteristic to be read. - // `options` - If null, a short read will be performed, which may be truncated - // to what fits in a single message (at least 22 bytes). If long read - // options are present, performs a long read with the indicated options. - // `result_callback` - called with the result of the read and the value of the - // characteristic if successful. - // - // This may fail with the following errors: - // kInvalidHandle - if `handle` is invalid - // kInvalidParameters - if `options is invalid - // kReadNotPermitted or kInsufficient* if the server rejects the request. - // kFailure if the server returns an error not covered by the above errors. + /// Reads the value of a characteristic. + /// @param handle The handle of the characteristic to be read. + /// @param options If null, a short read will be performed, which may be + /// truncated to what fits in a single message (at least 22 bytes). If long + /// read options are present, performs a long read with the indicated options. + /// @param result_callback called with the result of the read and the value of + /// the characteristic if successful. + /// @retval kInvalidHandle `handle` is invalid. + /// @retval kInvalidParameters `options` is invalid. + /// @retval kReadNotPermitted The server rejected the request. + /// @retval kInsufficient* The server rejected the request. + /// @retval kFailure The server returned an error not covered by the above. void ReadCharacteristic(Handle handle, std::optional<LongReadOptions> options, ReadCallback&& result_callback); - // Writes `value` to the characteristic with `handle` using the provided - // `options`. It is not recommended to send additional writes while a write - // is already in progress (the server may receive simultaneous writes in any - // order). - // - // Parameters: - // `handle` - Handle of the characteristic to be written to - // `value` - The value to be written. - // `options` - Options that apply to the write. - // - // This may fail with the following errors: - // kInvalidHandle - if `handle` is invalid - // kInvalidParameters - if `options is invalid - // kWriteNotPermitted or kInsufficient* if the server rejects the request. - // kFailure if the server returns an error not covered by the above errors. + /// Writes `value` to the characteristic with `handle` using the provided + /// `options`. It is not recommended to send additional writes while a write + /// is already in progress (the server may receive simultaneous writes in any + /// order). + /// + /// @param handle Handle of the characteristic to be written to + /// @param value The value to be written. + /// @param options Options that apply to the write. + /// @param result_callback Returns a result upon completion of the write. + /// @retval kInvalidHandle `handle` is invalid. + /// @retval kInvalidParameters`options is invalid. + /// @retval kWriteNotPermitted The server rejected the request. + /// @retval kInsufficient* The server rejected the request. + /// @retval kFailure The server returned an error not covered by the above + /// errors. void WriteCharacteristic(Handle handle, span<const std::byte> value, WriteOptions options, Function<void(Result<Error>)>&& result_callback); - // Reads the value of the characteristic descriptor with `handle` and - // returns it in the reply. - // `handle` - The descriptor handle to read. - // `options` - Options that apply to the read. - // `result_callback` - Returns a result containing the value of the descriptor - // on success. - // - // This may fail with the following errors: - // `kInvalidHandle` - if `handle` is invalid. - // `kInvalidParameters` - if `options` is invalid. - // `kReadNotPermitted` or `INSUFFICIENT_*` - if the server rejects the read - // request. `kFailure` - if the server returns an error. + /// Reads the value of the characteristic descriptor with `handle` and + /// returns it in the reply. + /// @param handle The descriptor handle to read. + /// @param options Options that apply to the read. + /// @param result_callback Returns a result containing the value of the + /// descriptor on success. + /// @retval kInvalidHandle `handle` is invalid. + /// @retval kInvalidParameters`options` is invalid. + /// @retval kReadNotPermitted + /// @retval kInsufficient* The server rejected the request. + /// @retval kFailure The server returned an error not covered by the above + /// errors. void ReadDescriptor(Handle handle, std::optional<LongReadOptions> options, ReadCallback&& result_callback); + /// Writes `value` to the descriptor with `handle` using the provided + /// `options`. It is not recommended to send additional writes while a write + /// is already in progress (the server may receive simultaneous writes in any + /// order). + /// + /// @param handle Handle of the descriptor to be written to + /// @param value The value to be written. + /// @param options Options that apply to the write. + /// @param result_callback Returns a result upon completion of the write. + /// @retval kInvalidHandle `handle` is invalid. + /// @retval kInvalidParameters `options is invalid + /// @retval kWriteNotPermitted The server rejected the request. + /// @retval kInsufficient* The server rejected the request. + /// @retval kFailure The server returned an error not covered by the above + /// errors. void WriteDescriptor(Handle handle, span<const std::byte> value, WriteOptions options, Function<void(Result<Error>)>&& result_callback); - // Subscribe to notifications & indications from the characteristic with - // the given `handle`. - // - // Either notifications or indications will be enabled depending on - // characteristic properties. Indications will be preferred if they are - // supported. This operation fails if the characteristic does not have the - // "notify" or "indicate" property. - // - // A write request will be issued to configure the characteristic for - // notifications/indications if it contains a Client Characteristic - // Configuration descriptor. This method fails if an error occurs while - // writing to the descriptor. - // - // On success, `notification_callback` will be called when - // the peer sends a notification or indication. Indications are - // automatically confirmed. - // - // Subscriptions can be canceled with `StopNotifications`. - // - // Parameters: - // `handle` - the handle of the characteristic to subscribe to. - // `notification_callback` - will be called with the values of - // notifications/indications when received. - // `result_callback` - called with the result of enabling - // notifications/indications. - // - // This may fail with the following errors: - // `kFailure` - the characteristic does not support notifications or - // indications. - // `kInvalidHandle` - `handle` is invalid. - // `kWriteNotPermitted`or `kInsufficient*` - descriptor write error. + /// Subscribe to notifications & indications from the characteristic with + /// the given `handle`. + /// + /// Either notifications or indications will be enabled depending on + /// characteristic properties. Indications will be preferred if they are + /// supported. This operation fails if the characteristic does not have the + /// "notify" or "indicate" property. + /// + /// A write request will be issued to configure the characteristic for + /// notifications/indications if it contains a Client Characteristic + /// Configuration descriptor. This method fails if an error occurs while + /// writing to the descriptor. + /// + /// On success, `notification_callback` will be called when + /// the peer sends a notification or indication. Indications are + /// automatically confirmed. + /// + /// Subscriptions can be canceled with `StopNotifications`. + /// + /// @param handle the handle of the characteristic to subscribe to. + /// @param notification_callback will be called with the values of + /// notifications/indications when received. + /// @param result_callback called with the result of enabling + /// notifications/indications. + /// @retval kFailure The characteristic does not support notifications or + /// indications. + /// @retval kInvalidHandle `handle` is invalid. + /// @retval kWriteNotPermitted CCC descriptor write error. + /// @retval Insufficient* CCC descriptor write error. void RegisterNotificationCallback( Handle handle, NotificationCallback&& notification_callback, Function<void(Result<Error>)>&& result_callback); - // Stops notifications for the characteristic with the given `handle`. + /// Stops notifications for the characteristic with the given `handle`. void StopNotifications(Handle handle); private: - // Disconnect from the remote service. This method is called by the - // ~RemoteService::Ptr() when it goes out of scope, the API client should - // never call this method. + /// Disconnect from the remote service. This method is called by the + /// ~RemoteService::Ptr() when it goes out of scope, the API client should + /// never call this method. void Disconnect(); public: - // Movable RemoteService smart pointer. The remote server will remain - // connected until the returned RemoteService::Ptr is destroyed. + /// Movable RemoteService smart pointer. The remote server will remain + /// connected until the returned RemoteService::Ptr is destroyed. using Ptr = internal::RaiiPtr<RemoteService, &RemoteService::Disconnect>; }; -// Represents a GATT client that interacts with services on a GATT server. +/// Represents a GATT client that interacts with services on a GATT server. class Client { public: - // Represents a remote GATT service. + /// Represents a remote GATT service. struct RemoteServiceInfo { - // Uniquely identifies this GATT service. + /// Uniquely identifies this GATT service. Handle handle; - // Indicates whether this is a primary or secondary service. + /// Indicates whether this is a primary or secondary service. bool primary; - // The UUID that identifies the type of this service. - // There may be multiple services with the same UUID. + /// The UUID that identifies the type of this service. + /// There may be multiple services with the same UUID. Uuid type; }; virtual ~Client() = default; - // Enumerates existing services found on the peer that this Client represents, - // and provides a stream of updates thereafter. Results can be filtered by - // specifying a list of UUIDs in `uuids`. To further interact with services, - // clients must obtain a RemoteService protocol by calling ConnectToService(). - // `uuid_allowlist` - The allowlist of UUIDs to filter services with. - // `updated_callback` - Will be called with services that are - // updated/modified. - // `removed_callback` - Called with the handles of services - // that have been removed. Note that handles may be reused. + /// Enumerates existing services found on the peer that this Client + /// represents, and provides a stream of updates thereafter. Results can be + /// filtered by specifying a list of UUIDs in `uuids`. To further interact + /// with services, clients must obtain a RemoteService protocol by calling + /// ConnectToService(). `uuid_allowlist` - The allowlist of UUIDs to filter + /// services with. `updated_callback` - Will be called with services that are + /// updated/modified. + /// `removed_callback` - Called with the handles of services + /// that have been removed. Note that handles may be reused. virtual void WatchServices( Vector<Uuid> uuid_allowlist, Function<void(RemoteServiceInfo)>&& updated_callback, Function<void(Handle)>&& removed_callback) = 0; - // Stops service watching if started by `WatchServices`. + /// Stops service watching if started by `WatchServices`. virtual void StopWatchingServices(); - // Connects to a RemoteService. Only 1 connection per service is allowed. - // `handle` - the handle of the service to connect to. - // - // This may fail with the following errors: - // kInvalidParameters - `handle` does not correspond to a known service. + /// Connects to a RemoteService. Only 1 connection per service is allowed. + /// `handle` - the handle of the service to connect to. + /// + /// This may fail with the following errors: + /// kInvalidParameters - `handle` does not correspond to a known service. virtual Result<Error, RemoteService::Ptr> ConnectToService(Handle handle) = 0; }; diff --git a/pw_bluetooth/public/pw_bluetooth/gatt/server.h b/pw_bluetooth/public/pw_bluetooth/gatt/server.h index 3cf6fc085..34e729131 100644 --- a/pw_bluetooth/public/pw_bluetooth/gatt/server.h +++ b/pw_bluetooth/public/pw_bluetooth/gatt/server.h @@ -28,208 +28,236 @@ namespace pw::bluetooth::gatt { -// Parameters for registering a local GATT service. +/// Parameters for registering a local GATT service. struct LocalServiceInfo { - // A unique (within a Server) handle identifying this service. + /// A unique (within a Server) handle identifying this service. Handle handle; - // Indicates whether this is a primary or secondary service. + /// Indicates whether this is a primary or secondary service. bool primary; - // The UUID that identifies the type of this service. - // There may be multiple services with the same UUID. + /// The UUID that identifies the type of this service. + /// There may be multiple services with the same UUID. Uuid type; - // The characteristics of this service. + /// The characteristics of this service. span<const Characteristic> characteristics; - // Handles of other services that are included by this service. + /// Handles of other services that are included by this service. span<const Handle> includes; }; -// Interface for serving a local GATT service. This is implemented by the API -// client. +/// Interface for serving a local GATT service. This is implemented by the API +/// client. class LocalServiceDelegate { public: virtual ~LocalServiceDelegate() = default; - // Called when there is a fatal error related to this service that forces the - // service to close. LocalServiceDelegate methods will no longer be called. - // This invalidates the associated LocalService. It is OK to destroy both - // `LocalServiceDelegate` and the associated `LocalService::Ptr` from within - // this method. + /// Called when there is a fatal error related to this service that forces the + /// service to close. LocalServiceDelegate methods will no longer be called. + /// This invalidates the associated LocalService. It is OK to destroy both + /// `LocalServiceDelegate` and the associated `LocalService::Ptr` from within + /// this method. virtual void OnError(Error error) = 0; - // This notifies the current configuration of a particular - // characteristic/descriptor for a particular peer. It will be called when the - // peer GATT client changes the configuration. - // - // The Bluetooth stack maintains the state of each peer's configuration across - // reconnections. As such, this method will also be called when a peer - // connects for each characteristic with the initial, persisted state of the - // newly-connected peer's configuration. However, clients should not rely on - // this state being persisted indefinitely by the Bluetooth stack. - // - // Parameters: - // `peer_id` - The PeerId of the GATT client associated with this particular - // CCC. - // `handle` - The handle of the characteristic associated with the `notify` - // and `indicate` parameters. - // `notify` - True if the client has enabled notifications, false otherwise. - // `indicate` - True if the client has enabled indications, false otherwise. + /// This notifies the current configuration of a particular + /// characteristic/descriptor for a particular peer. It will be called when + /// the peer GATT client changes the configuration. + /// + /// The Bluetooth stack maintains the state of each peer's configuration + /// across reconnections. As such, this method will be called with both + /// `notify` and `indicate` set to false for each characteristic when a peer + /// disconnects. Also, when a peer reconnects this method will be called again + /// with the initial, persisted state of the newly-connected peer's + /// configuration. However, clients should not rely on this state being + /// persisted indefinitely by the Bluetooth stack. + /// + /// @param peer_id The PeerId of the GATT client associated with this + /// particular CCC. + /// @param handle The handle of the characteristic associated with the + /// `notify` and `indicate` parameters. + /// @param notify True if the client has enabled notifications, false + /// otherwise. + /// @param indicate True if the client has enabled indications, false + /// otherwise. virtual void CharacteristicConfiguration(PeerId peer_id, Handle handle, bool notify, bool indicate) = 0; - // Called when a peer requests to read the value of a characteristic or - // descriptor. It is guaranteed that the peer satisfies the permissions - // associated with this attribute. - // - // Parameters: - // `peer_id` - The PeerId of the GATT client making the read request. - // `handle` - The handle of the requested descriptor/characteristic. - // `offset` - The offset at which to start reading the requested value. - // `result_callback` - Called with the value of the characteristic on success, - // or an Error on failure. The value will be truncated to fit in the MTU - // if necessary. It is OK to call `result_callback` in `ReadValue`. + /// Called when a peer requests to read the value of a characteristic or + /// descriptor. It is guaranteed that the peer satisfies the permissions + /// associated with this attribute. + /// + /// @param peer_id The PeerId of the GATT client making the read request. + /// @param handle The handle of the requested descriptor/characteristic. + /// @param offset The offset at which to start reading the requested value. + /// @param result_callback Called with the value of the characteristic on + /// success, or an Error on failure. The value will be truncated to fit in the + /// MTU if necessary. It is OK to call `result_callback` in `ReadValue`. virtual void ReadValue(PeerId peer_id, Handle handle, uint32_t offset, Function<void(Result<Error, span<const std::byte>>)>&& result_callback) = 0; - // Called when a peer issues a request to write the value of a characteristic - // or descriptor. It is guaranteed that the peer satisfies the permissions - // associated with this attribute. - // - // Parameters: - // `peer_id` - The PeerId of the GATT client making the write request. - // `handle` - The handle of the requested descriptor/characteristic. - // `offset` - The offset at which to start writing the requested value. If the - // offset is 0, any existing value should be overwritten by the new value. - // Otherwise, the existing value between offset:(offset + len(value)) - // should be changed to `value`. - // `value` - The new value for the descriptor/characteristic. - // `status_callback` - Called with the result of the write. + /// Called when a peer issues a request to write the value of a characteristic + /// or descriptor. It is guaranteed that the peer satisfies the permissions + /// associated with this attribute. + /// + /// @param peer_id The PeerId of the GATT client making the write request. + /// @param handle The handle of the requested descriptor/characteristic. + /// @param offset The offset at which to start writing the requested value. If + /// the offset is 0, any existing value should be overwritten by the new + /// value. Otherwise, the existing value between `offset:(offset + + /// len(value))` should be changed to `value`. + /// @param value The new value for the descriptor/characteristic. + /// @param status_callback Called with the result of the write. virtual void WriteValue(PeerId peer_id, Handle handle, uint32_t offset, span<const std::byte> value, Function<void(Result<Error>)>&& status_callback) = 0; - // Called when the MTU of a peer is updated. Also called for peers that are - // already connected when the server is published. This method is safe to - // ignore if you do not care about the MTU. It is intended for use cases where - // throughput needs to be optimized. + /// Called when the MTU of a peer is updated. Also called for peers that are + /// already connected when the server is published. + /// + /// Notifications and indications must fit in a single packet including both + /// the 3-byte notification/indication header and the user-provided payload. + /// If these are not used, the MTU can be safely ignored as it is intended for + /// use cases where the throughput needs to be optimized. virtual void MtuUpdate(PeerId peer_id, uint16_t mtu) = 0; }; -// Interface provided by the backend to interact with a published service. -// LocalService is valid for the lifetime of a published GATT service. It is -// used to control the service and send notifications/indications. +/// Interface provided by the backend to interact with a published service. +/// LocalService is valid for the lifetime of a published GATT service. It is +/// used to control the service and send notifications/indications. class LocalService { public: - // The parameters used to signal a characteristic value change from a - // LocalService to a peer. + /// The parameters used to signal a characteristic value change from a + /// LocalService to a peer. struct ValueChangedParameters { - // The PeerIds of the peers to signal. The LocalService should respect the - // Characteristic Configuration associated with a peer+handle when deciding - // whether to signal it. If empty, all peers are signalled. + /// The PeerIds of the peers to signal. The LocalService should respect the + /// Characteristic Configuration associated with a peer+handle when deciding + /// whether to signal it. If empty, all peers are signalled. span<const PeerId> peer_ids; - // The handle of the characteristic value being signaled. + /// The handle of the characteristic value being signaled. Handle handle; - // The new value for the descriptor/characteristic. + /// The new value for the descriptor/characteristic. span<const std::byte> value; }; + /// The Result type for a ValueChanged indication or notification message. The + /// error can be locally generated for notifications and either locally or + /// remotely generated for indications. + using ValueChangedResult = Result<Error>; + + /// The callback type for a ValueChanged indication or notification + /// completion. + using ValueChangedCallback = Function<void(ValueChangedResult)>; + virtual ~LocalService() = default; - // Sends a notification to peers. Notifications should be used instead of - // indications when the service does *not* require peer confirmation of the - // update. - // - // Notifications should not be sent to peers which have not enabled - // notifications on a particular characteristic - if they are sent, they will - // not be propagated. The Bluetooth stack will track this configuration for - // the lifetime of the service. - // - // Parameters: - // `parameters` - The parameters associated with the changed characteristic. - // `completion_callback` - Called when the notification has been sent. - // Additional values should not be notified until this callback is called. + /// Sends a notification to peers. Notifications should be used instead of + /// indications when the service does *not* require peer confirmation of the + /// update. + /// + /// Notifications should not be sent to peers which have not enabled + /// notifications on a particular characteristic or that have disconnected + /// since - if they are sent, they will not be propagated and the + /// `completion_callback` will be called with an error condition. The + /// Bluetooth stack will track this configuration for the lifetime of the + /// service. + /// + /// The maximum size of the `parameters.value` field depends on the MTU + /// negotiated with the peer. A 3-byte header plus the value contents must fit + /// in a packet of MTU bytes. + /// + /// @param parameters The parameters associated with the changed + /// characteristic. + /// @param completion_callback Called when the notification has been sent to + /// all peers or an error is produced when trying to send the notification to + /// any of the peers. This function is called only once when all associated + /// work is done, if the implementation wishes to receive a call on a + /// per-peer basis, they should send this event with a single PeerId in + /// `parameters.peer_ids`. Additional values should not be notified until + /// this callback is called. virtual void NotifyValue(const ValueChangedParameters& parameters, - Closure&& completion_callback) = 0; - - // Sends an indication to peers. Indications should be used instead of - // notifications when the service *does* require peer confirmation of the - // update. - // - // Indications should not be sent to peers which have not enabled indications - // on a particular characteristic - if they are sent, they will not be - // propagated. The Bluetooth stack will track this configuration for the - // lifetime of the service. - // - // If any of the peers in `update.peer_ids` fails to confirm the indication - // within the ATT transaction timeout (30 seconds per Bluetooth 5.2 Vol. 4 - // Part G 3.3.3), the link between the peer and the local adapter will be - // closed. - // - // Parameters: - // `parameters` - The parameters associated with the changed characteristic. - // `confirmation` - When all the peers listed in `parameters.peer_ids` have - // confirmed the indication, `confirmation` is called. If the - // implementation wishes to receive indication confirmations on a per-peer - // basis, they should send this event with a single PeerId in - // `parameters.peer_ids`. Additional values should not be indicated until - // this callback is called. + ValueChangedCallback&& completion_callback) = 0; + + /// Sends an indication to peers. Indications should be used instead of + /// notifications when the service *does* require peer confirmation of the + /// update. + /// + /// Indications should not be sent to peers which have not enabled indications + /// on a particular characteristic - if they are sent, they will not be + /// propagated. The Bluetooth stack will track this configuration for the + /// lifetime of the service. + /// + /// If any of the peers in `parameters.peer_ids` fails to confirm the + /// indication within the ATT transaction timeout (30 seconds per + /// Bluetooth 5.2 Vol. 4 Part G 3.3.3), the link between the peer and the + /// local adapter will be closed. + /// + /// The maximum size of the `parameters.value` field depends on the MTU + /// negotiated with the peer. A 3-byte header plus the value contents must fit + /// in a packet of MTU bytes. + /// + /// @param parameters The parameters associated with the changed + /// characteristic. + /// @param confirmation When all the peers listed in `parameters.peer_ids` + /// have confirmed the indication, `confirmation` is called. If the + /// implementation wishes to receive indication confirmations on a per-peer + /// basis, they should send this event with a single PeerId in + /// `parameters.peer_ids`. Additional values should not be indicated until + /// this callback is called. virtual void IndicateValue(const ValueChangedParameters& parameters, - Function<void(Result<Error>)>&& confirmation) = 0; + ValueChangedCallback&& confirmation) = 0; private: - // Unpublish the local service. This method is called by the - // ~LocalService::Ptr() when it goes out of scope, the API client should never - // call this method. + /// Unpublish the local service. This method is called by the + /// ~LocalService::Ptr() when it goes out of scope, the API client should + /// never call this method. virtual void UnpublishService() = 0; public: - // Movable LocalService smart pointer. When the LocalService::Ptr object is - // destroyed the service will be unpublished. + /// Movable LocalService smart pointer. When the LocalService::Ptr object is + /// destroyed the service will be unpublished. using Ptr = internal::RaiiPtr<LocalService, &LocalService::UnpublishService>; }; -// Interface for a GATT server that serves many GATT services. +/// Interface for a GATT server that serves many GATT services. class Server { public: enum class PublishServiceError { kInternalError = 0, - // The service handle provided was not unique. + /// The service handle provided was not unique. kInvalidHandle = 1, - // Invalid service UUID provided. + /// Invalid service UUID provided. kInvalidUuid = 2, - // Invalid service characteristics provided. + /// Invalid service characteristics provided. kInvalidCharacteristics = 3, - // Invalid service includes provided. + /// Invalid service includes provided. kInvalidIncludes = 4, }; - // The Result passed by PublishService. + /// The Result passed by PublishService. using PublishServiceResult = Result<PublishServiceError, LocalService::Ptr>; virtual ~Server() = default; - // Publishes the service defined by `info` and implemented by `delegate` so - // that it is available to all remote peers. - // - // The caller must assign distinct handles to the characteristics and - // descriptors listed in `info`. These identifiers will be used in requests - // sent to `delegate`. On success, a `LocalService::Ptr` is returned. When the - // `LocalService::Ptr` is destroyed or an error occurs - // (LocalServiceDelegate.OnError), the service will be unpublished. + /// Publishes the service defined by `info` and implemented by `delegate` so + /// that it is available to all remote peers. + /// + /// The caller must assign distinct handles to the characteristics and + /// descriptors listed in `info`. These identifiers will be used in requests + /// sent to `delegate`. On success, a `LocalService::Ptr` is returned. When + /// the `LocalService::Ptr` is destroyed or an error occurs + /// (LocalServiceDelegate.OnError), the service will be unpublished. virtual void PublishService( const LocalServiceInfo& info, LocalServiceDelegate* delegate, diff --git a/pw_bluetooth/public/pw_bluetooth/hci.emb b/pw_bluetooth/public/pw_bluetooth/hci.emb index ed2267979..d781b3f18 100644 --- a/pw_bluetooth/public/pw_bluetooth/hci.emb +++ b/pw_bluetooth/public/pw_bluetooth/hci.emb @@ -520,23 +520,29 @@ bits ClassOfDevice: enum LEPeriodicAdvertisingCreateSyncUseParams: [maximum_bits: 1] + USE_PARAMS = 0x00 -- Use the Advertising_SID, Advertiser_Address_Type, and Adertiser_Address parameters to -- determine which advertiser to listen to. + USE_PERIODIC_ADVERTISER_LIST = 0x01 -- Use the Periodic Advertiser List to determine which advertiser to listen to. bits LEPeriodicAdvertisingCreateSyncOptions: -- First parameter to the LE Periodic Advertising Create Sync command - 0 [+1] LEPeriodicAdvertisingCreateSyncUseParams advertiser_source - $next [+1] Flag enable_reporting + + 0 [+1] LEPeriodicAdvertisingCreateSyncUseParams advertiser_source + + $next [+1] Flag enable_reporting -- 0: Reporting initially enabled -- 1: Reporting initially disabled - $next [+1] Flag enable_duplicate_filtering + + $next [+1] Flag enable_duplicate_filtering -- 0: Duplicate filtering initially disabled -- 1: Duplicate filtering initially enabled - $next [+5] UInt padding + + $next [+5] UInt padding -- Reserved for future use @@ -546,23 +552,30 @@ enum LEPeriodicAdvertisingAddressType: [maximum_bits: 8] PUBLIC = 0x00 -- Public Device Address or Public Identity Address + RANDOM = 0x01 -- Random Device Address or Random (static) Identity Address bits LEPeriodicAdvertisingSyncCTEType: -- Bit definitions for a |sync_cte_type| field in an LE Periodic Advertising Create Sync command + 0 [+1] Flag dont_sync_aoa -- Do not sync to packets with an AoA Constant Tone Extension + $next [+1] Flag dont_sync_aod_1us -- Do not sync to packets with an AoD Constant Tone Extension with 1 microsecond slots + $next [+1] Flag dont_sync_aod_2us -- Do not sync to packets with an AoD Constant Tone Extension with 2 microsecond slots + $next [+1] Flag dont_sync_type_3 -- Do not sync to packets with a typoe 3 Constant Tone Extension (currently reserved for future -- use) + $next [+1] Flag dont_sync_without_cte -- Do not sync to packets without a Constant Tone Extension + $next [+3] UInt padding -- Reserved for future use @@ -597,14 +610,19 @@ enum LEAddressType: enum LEOwnAddressType: -- Possible values that can be used for the |own_address_type| parameter in various HCI commands + [maximum_bits: 8] + PUBLIC = 0x00 -- Public Device Address + RANDOM = 0x01 -- Random Device Address + PRIVATE_DEFAULT_TO_PUBLIC = 0x02 -- Controller generates the Resolvable Private Address based on the local IRK from the resolving -- list. If the resolving list contains no matching entry, then use the public address. + PRIVATE_DEFAULT_TO_RANDOM = 0x03 -- Controller generates the Resolvable Private Address based on the local IRK from the resolving -- list. If the resolving list contains no matching entry, then use the random address from @@ -625,6 +643,7 @@ enum LEScanType: [maximum_bits: 8] PASSIVE = 0x00 -- Passive Scanning. No scanning PDUs shall be sent (default) + ACTIVE = 0x01 -- Active scanning. Scanning PDUs may be sent. @@ -639,15 +658,19 @@ enum LEScanFilterPolicy: bits LEPHYBits: - 0 [+1] Flag le_1m + 0 [+1] Flag le_1m -- Scan advertisements on the LE 1M PHY - $next [+1] Flag padding1 + + $next [+1] Flag padding1 -- Reserved for future use - $next [+1] Flag le_coded + + $next [+1] Flag le_coded -- Scan advertisements on the LE Coded PHY - $next [+5] UInt padding2 + + $next [+5] UInt padding2 -- Reserved for future use + enum LEPrivacyMode: -- Possible values for the |privacy_mode| parameter in an LE Set Privacy Mode -- command @@ -679,6 +702,251 @@ enum PageScanType: INTERLACED_SCAN = 0x01 -- Interlaced scan (optional) + +bits LEEventMask: + 0 [+1] Flag le_connection_complete + $next [+1] Flag le_advertising_report + $next [+1] Flag le_connection_update_complete + $next [+1] Flag le_read_remote_features_complete + $next [+1] Flag le_long_term_key_request + $next [+1] Flag le_remote_connection_parameter_request + $next [+1] Flag le_data_length_change + $next [+1] Flag le_read_local_p256_public_key_complete + $next [+1] Flag le_generate_dhkey_complete + $next [+1] Flag le_enhanced_connection_complete + $next [+1] Flag le_directed_advertising_report + $next [+1] Flag le_phy_update_complete + $next [+1] Flag le_extended_advertising_report + $next [+1] Flag le_periodic_advertising_sync_established + $next [+1] Flag le_periodic_advertising_report + $next [+1] Flag le_periodic_advertising_sync_lost + $next [+1] Flag le_extended_scan_timeout + $next [+1] Flag le_extended_advertising_set_terminated + $next [+1] Flag le_scan_request_received + $next [+1] Flag le_channel_selection_algorithm + $next [+1] Flag le_connectionless_iq_report + $next [+1] Flag le_connection_iq_report + $next [+1] Flag le_cte_request_failed + $next [+1] Flag le_periodic_advertising_sync_transfer_received_event + $next [+1] Flag le_cis_established_event + $next [+1] Flag le_cis_request_event + $next [+1] Flag le_create_big_complete_event + $next [+1] Flag le_terminate_big_complete_event + $next [+1] Flag le_big_sync_established_event + $next [+1] Flag le_big_sync_lost_event + $next [+1] Flag le_request_peer_sca_complete_event + $next [+1] Flag le_path_loss_threshold_event + $next [+1] Flag le_transmit_power_reporting_event + $next [+1] Flag le_biginfo_advertising_report_event + $next [+1] Flag le_subrate_change_event + + +enum LEAdvertisingType: + [maximum_bits: 8] + CONNECTABLE_AND_SCANNABLE_UNDIRECTED = 0x00 + -- ADV_IND + + CONNECTABLE_HIGH_DUTY_CYCLE_DIRECTED = 0x01 + -- ADV_DIRECT_IND + + SCANNABLE_UNDIRECTED = 0x02 + -- ADV_SCAN_IND + + NOT_CONNECTABLE_UNDIRECTED = 0x03 + -- ADV_NONCONN_IND + + CONNECTABLE_LOW_DUTY_CYCLE_DIRECTED = 0x04 + -- ADV_DIRECT_IND + + +bits LEAdvertisingChannels: + 0 [+1] Flag channel_37 + $next [+1] Flag channel_38 + $next [+1] Flag channel_39 + + +enum LEAdvertisingFilterPolicy: + [maximum_bits: 8] + + ALLOW_ALL = 0x00 + -- Process scan and connection requests from all devices (i.e., the Filter + -- Accept List is not in use) (default). + + ALLOW_ALL_CONNECTIONS_AND_USE_FILTER_ACCEPT_LIST_FOR_SCANS = 0x01 + -- Process connection requests from all devices and scan requests only from + -- devices that are in the Filter Accept List. + + ALLOW_ALL_SCANS_AND_USE_FILTER_ACCEPT_LIST_FOR_CONNECTIONS = 0x02 + -- Process scan requests from all devices and connection requests only from + -- devices that are in the Filter Accept List. + + ALLOW_FILTER_ACCEPT_LIST_ONLY = 0x03 + -- Process scan and connection requests only from devices in the Filter + -- Accept List. + + +enum LESetExtendedAdvDataOp: + -- Potential values for the Operation parameter in a HCI_LE_Set_Extended_Advertising_Data command. + [maximum_bits: 8] + INTERMEDIATE_FRAGMENT = 0x00 + -- Intermediate fragment of fragmented extended advertising data. + + FIRST_FRAGMENT = 0x01 + -- First fragment of fragmented extended advertising data. + + LAST_FRAGMENT = 0x02 + -- Last fragment of fragmented extended advertising data. + + COMPLETE = 0x03 + -- Complete extended advertising data. + + UNCHANGED_DATA = 0x04 + -- Unchanged data (just update the Advertising DID) + + +enum LEExtendedAdvFragmentPreference: + -- Potential values for the Fragment_Preference parameter in a + -- HCI_LE_Set_Extended_Advertising_Data command. + [maximum_bits: 8] + MAY_FRAGMENT = 0x00 + -- The Controller may fragment all Host advertising data + + SHOULD_NOT_FRAGMENT = 0x01 + -- The Controller should not fragment or should minimize fragmentation of Host advertising data + + +enum FlowControlMode: + [maximum_bits: 8] + PACKET_BASED = 0x00 + DATA_BLOCK_BASED = 0x01 + + +bits EventMaskPage2: + 8 [+1] Flag number_of_completed_data_blocks_event + 14 [+1] Flag triggered_clock_capture_event + 15 [+1] Flag synchronization_train_complete_event + 16 [+1] Flag synchronization_train_received_event + 17 [+1] Flag connectionless_peripheral_broadcast_receive_event + 18 [+1] Flag connectionless_peripheral_broadcast_timeout_event + 19 [+1] Flag truncated_page_complete_event + 20 [+1] Flag peripheral_page_response_timeout_event + 21 [+1] Flag connectionless_peripheral_broadcast_channel_map_event + 22 [+1] Flag inquiry_response_notification_event + 23 [+1] Flag authenticated_payload_timeout_expired_event + 24 [+1] Flag sam_status_change_event + 25 [+1] Flag encryption_change_event_v2 + + +enum LinkType: + [maximum_bits: 8] + SCO = 0x00 + ACL = 0x01 + ESCO = 0x02 + + +enum EncryptionStatus: + OFF = 0x00 + ON_WITH_E0_FOR_BREDR_OR_AES_FOR_LE = 0x01 + ON_WITH_AES_FOR_BREDR = 0x03 + + +bits LmpFeatures(page: UInt:8): + -- Bit mask of Link Manager Protocol features. + if page == 0: + 0 [+1] Flag three_slot_packets + 1 [+1] Flag five_slot_packets + 2 [+1] Flag encryption + 3 [+1] Flag slot_offset + 4 [+1] Flag timing_accuracy + 5 [+1] Flag role_switch + 6 [+1] Flag hold_mode + 7 [+1] Flag sniff_mode + # 8: previously used + 9 [+1] Flag power_control_requests + 10 [+1] Flag channel_quality_driven_data_rate + 11 [+1] Flag sco_link + 12 [+1] Flag hv2_packets + 13 [+1] Flag hv3_packets + 14 [+1] Flag mu_law_log_synchronous_data + 15 [+1] Flag a_law_log_synchronous_data + 16 [+1] Flag cvsd_synchronous_data + 17 [+1] Flag paging_parameter_negotiation + 18 [+1] Flag power_control + 19 [+1] Flag transparent_synchronous_data + 20 [+3] UInt flow_control_lag + 23 [+1] Flag broadcast_encryption + # 24: reserved for future use + 25 [+1] Flag enhanced_data_rate_acl_2_mbs_mode + 26 [+1] Flag enhanced_data_rate_acl_3_mbs_mode + 27 [+1] Flag enhanced_inquiry_scan + 28 [+1] Flag interlaced_inquiry_scan + 29 [+1] Flag interlaced_page_scan + 30 [+1] Flag rssi_with_inquiry_results + 31 [+1] Flag extended_sco_link_ev3_packets + 32 [+1] Flag ev4_packets + 33 [+1] Flag ev5_packets + # 34: reserved for future use + 35 [+1] Flag afh_capable_peripheral + 36 [+1] Flag afh_classification_peripheral + 37 [+1] Flag bredr_not_supported + 38 [+1] Flag le_supported_controller + 39 [+1] Flag three_slot_enhanced_data_rate_acl_packets + 40 [+1] Flag five_slot_enhanced_data_rate_acl_packets + 41 [+1] Flag sniff_subrating + 42 [+1] Flag pause_encryption + 43 [+1] Flag afh_capable_central + 44 [+1] Flag afh_classification_central + 45 [+1] Flag enhanced_data_rate_esco_2_mbs_mode + 46 [+1] Flag enhanced_data_rate_esco_3_mbs_mode + 47 [+1] Flag three_slot_enhanced_data_rate_esco_packets + 48 [+1] Flag extended_inquiry_response + 49 [+1] Flag simultaneous_le_and_bredr_to_same_device_capable_controller + # 50: reserved for future use + 51 [+1] Flag secure_simple_pairing_controller_support + 52 [+1] Flag encapsulated_pdu + 53 [+1] Flag erroneous_data_reporting + 54 [+1] Flag non_flushable_packet_boundary_flag + # 55: reserved for future use + 56 [+1] Flag hci_link_supervision_timeout_changed_event + 57 [+1] Flag variable_inquiry_tx_power_level + 58 [+1] Flag enhanced_power_control + # 59-62: reserved for future use + 63 [+1] Flag extended_features + + if page == 1: + 0 [+1] Flag secure_simple_pairing_host_support + 1 [+1] Flag le_supported_host + # 2: previously used + 3 [+1] Flag secure_connection_host_support + + if page == 2: + 0 [+1] Flag connectionless_peripheral_broadcast_transmitter_operation + 1 [+1] Flag connectionless_peripheral_broadcast_receiver_operation + 2 [+1] Flag synchronization_train + 3 [+1] Flag synchronization_scan + 4 [+1] Flag hci_inquiry_response_notification_event + 5 [+1] Flag generalized_interlaced_scan + 6 [+1] Flag coarse_clock_adjustment + # 7: reserved for future use + 8 [+1] Flag secure_connections_controller_support + 9 [+1] Flag ping + 10 [+1] Flag slot_availability_mask + 11 [+1] Flag train_nudging + + +enum LEClockAccuracy: + -- Possible values that can be reported for the |central_clock_accuracy| and + -- |advertiser_clock_accuracy| parameters. + [maximum_bits: 8] + PPM_500 = 0x00 + PPM_250 = 0x01 + PPM_150 = 0x02 + PPM_100 = 0x03 + PPM_75 = 0x04 + PPM_50 = 0x05 + PPM_30 = 0x06 + PPM_20 = 0x07 + # ========================= HCI packet headers ========================== @@ -1263,6 +1531,52 @@ struct LESetExtendedAdvertisingEnableData: $next [+1] UInt max_extended_advertising_events +struct LESetExtendedAdvertisingDataCommand: + -- LE Set Extended Advertising Data Command (v5.0) (LE) + + let hdr_size = CommandHeader.$size_in_bytes + + 0 [+hdr_size] CommandHeader header + + $next [+1] UInt advertising_handle + -- Handle used to identify an advertising set. + + $next [+1] LESetExtendedAdvDataOp operation + + $next [+1] LEExtendedAdvFragmentPreference fragment_preference + -- Provides a hint to the Controller as to whether advertising data should be fragmented. + + $next [+1] UInt advertising_data_length (sz) + -- Length of the advertising data included in this command packet, up to + -- kMaxLEExtendedAdvertisingDataLength bytes. If the advertising set uses legacy advertising + -- PDUs that support advertising data then this shall not exceed kMaxLEAdvertisingDataLength + -- bytes. + [requires: 0 <= this <= 251] + + $next [+sz] UInt:8[sz] advertising_data + -- Variable length advertising data. + + +struct LESetExtendedScanResponseDataCommand: + -- LE Set Extended Scan Response Data Command (v5.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+1] UInt advertising_handle + -- Used to identify an advertising set + [requires: 0x00 <= this <= 0xEF] + + $next [+1] LESetExtendedAdvDataOp operation + $next [+1] LEExtendedAdvFragmentPreference fragment_preference + -- Provides a hint to the controller as to whether advertising data should be fragmented + + $next [+1] UInt scan_response_data_length (sz) + -- The number of octets in the scan_response_data parameter + [requires: 0 <= this <= 251] + + $next [+sz] UInt:8[sz] scan_response_data + -- Scan response data formatted as defined in Core Spec v5.4, Vol 3, Part C, Section 11 + + struct LESetExtendedAdvertisingEnableCommand: -- LE Set Extended Advertising Enable command (v5.0) (LE) let hdr_size = CommandHeader.$size_in_bytes @@ -1273,6 +1587,20 @@ struct LESetExtendedAdvertisingEnableCommand: $next [+single_data_size*num_sets] LESetExtendedAdvertisingEnableData[] data +struct LEReadMaxAdvertisingDataLengthCommand: + -- LE Read Maximum Advertising Data Length Command (v5.0) (LE) + -- This command has no parameters + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + +struct LEReadNumSupportedAdvertisingSetsCommand: + -- LE Read Number of Supported Advertising Sets Command (v5.0) (LE) + -- This command has no parameters + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + struct LERemoveAdvertisingSetCommand: -- LE Remove Advertising Set command (v5.0) (LE) let hdr_size = CommandHeader.$size_in_bytes @@ -1280,16 +1608,26 @@ struct LERemoveAdvertisingSetCommand: $next [+1] UInt advertising_handle +struct LEClearAdvertisingSetsCommand: + -- LE Clear Advertising Sets Command (v5.0) (LE) + -- This command has no parameters + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + struct LESetExtendedScanParametersData: -- Data fields for variable-length portion of an LE Set Extneded Scan Parameters command - 0 [+1] LEScanType scan_type - $next [+2] UInt scan_interval + + 0 [+1] LEScanType scan_type + + $next [+2] UInt scan_interval -- Time interval from when the Controller started its last scan until it begins the subsequent -- scan on the primary advertising physical channel. -- Time = N × 0.625 ms -- Time Range: 2.5 ms to 40.959375 s [requires: 0x0004 <= this] - $next [+2] UInt scan_window + + $next [+2] UInt scan_window -- Duration of the scan on the primary advertising physical channel. -- Time = N × 0.625 ms -- Time Range: 2.5 ms to 40.959375 s @@ -1300,13 +1638,13 @@ struct LESetExtendedScanParametersCommand(num_entries: UInt:8): -- LE Set Extended Scan Parameters Command (v5.0) (LE) -- num_entries corresponds to the number of bits set in the |scanning_phys| field let hdr_size = CommandHeader.$size_in_bytes - 0 [+hdr_size] CommandHeader header - $next [+1] LEOwnAddressType own_address_type - $next [+1] LEScanFilterPolicy scanning_filter_policy - $next [+1] LEPHYBits scanning_phys + 0 [+hdr_size] CommandHeader header + $next [+1] LEOwnAddressType own_address_type + $next [+1] LEScanFilterPolicy scanning_filter_policy + $next [+1] LEPHYBits scanning_phys let single_entry_size = LESetExtendedScanParametersData.$size_in_bytes - let total_entries_size = num_entries * single_entry_size - $next [+total_entries_size] LESetExtendedScanParametersData[num_entries] data + let total_entries_size = num_entries*single_entry_size + $next [+total_entries_size] LESetExtendedScanParametersData[num_entries] data -- Indicates the type of address being used in the scan request packets (for active scanning). @@ -1402,26 +1740,35 @@ struct WriteClassOfDeviceCommand: struct LEPeriodicAdvertisingCreateSyncCommand: -- LE Periodic Advertising Create Sync Command (v5.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes - 0 [+hdr_size] CommandHeader header - $next [+1] LEPeriodicAdvertisingCreateSyncOptions options - $next [+1] UInt advertising_sid + + 0 [+hdr_size] CommandHeader header + + $next [+1] LEPeriodicAdvertisingCreateSyncOptions options + + $next [+1] UInt advertising_sid -- Advertising SID subfield in the ADI field used to identify the Periodic Advertising [requires: 0x00 <= this <= 0x0F] - $next [+1] LEPeriodicAdvertisingAddressType advertiser_address_type - $next [+BdAddr.$size_in_bytes] BdAddr advertiser_address + + $next [+1] LEPeriodicAdvertisingAddressType advertiser_address_type + + $next [+BdAddr.$size_in_bytes] BdAddr advertiser_address -- Public Device Address, Random Device Address, Public Identity Address, or Random (static) -- Identity Address of the advertiser - $next [+2] UInt skip + + $next [+2] UInt skip -- The maximum number of periodic advertising events that can be skipped after a successful -- receive [requires: 0x0000 <= this <= 0x01F3] - $next [+2] UInt sync_timeout + + $next [+2] UInt sync_timeout -- Synchronization timeout for the periodic advertising. -- Time = N * 10 ms -- Time Range: 100 ms to 163.84 s [requires: 0x000A <= this <= 0x4000] - $next [+1] LEPeriodicAdvertisingSyncCTEType sync_cte_type + + $next [+1] LEPeriodicAdvertisingSyncCTEType sync_cte_type -- Constant Tone Extension sync options @@ -1429,14 +1776,14 @@ struct LEPeriodicAdvertisingCreateSyncCancel: -- LE Periodic Advertising Create Sync Cancel Command (v5.0) (LE) -- Note that this command has no arguments let hdr_size = CommandHeader.$size_in_bytes - 0 [+hdr_size] CommandHeader header + 0 [+hdr_size] CommandHeader header struct LEPeriodicAdvertisingTerminateSyncCommand: -- LE Periodic Advertising Terminate Sync Command (v5.0) (LE) let hdr_size = CommandHeader.$size_in_bytes - 0 [+hdr_size] CommandHeader header - $next [+2] UInt sync_handle + 0 [+hdr_size] CommandHeader header + $next [+2] UInt sync_handle -- Identifies the periodic advertising train [requires: 0x0000 <= this <= 0x0EFF] @@ -1564,6 +1911,464 @@ struct ReadSimplePairingModeCommand: let hdr_size = CommandHeader.$size_in_bytes 0 [+hdr_size] CommandHeader header + +struct WriteLEHostSupportCommand: + -- Write LE Host Support Command (v4.0) (BR/EDR) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+1] GenericEnableParam le_supported_host + -- Sets the LE Supported (Host) Link Manager Protocol feature bit. + + $next [+1] UInt unused + -- Core Spec v5.0, Vol 2, Part E, Section 6.35: This parameter was named + -- "Simultaneous_LE_Host" and the value is set to "disabled(0x00)" and + -- "shall be ignored". + -- Core Spec v5.3, Vol 4, Part E, Section 7.3.79: This parameter was renamed + -- to "Unused" and "shall be ignored by the controller". + + +struct ReadLocalVersionInformationCommand: + -- Read Local Version Information Command (v1.1) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + +struct ReadLocalSupportedCommandsCommand: + -- Read Local Supported Commands Command (v1.2) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + +struct ReadBufferSizeCommand: + -- Read Buffer Size Command (v1.1) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + +struct ReadBdAddrCommand: + -- Read BD_ADDR Command (v1.1) (BR/EDR, LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + +struct ReadLocalSupportedFeaturesCommand: + -- Read Local Supported Features Command (v1.1) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + +struct ReadLocalExtendedFeaturesCommand: + -- Read Local Extended Features Command (v1.2) (BR/EDR) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+1] UInt page_number + -- 0x00: Requests the normal LMP features as returned by + -- Read_Local_Supported_Features. + -- 0x01-0xFF: Return the corresponding page of features. + + +struct ReadEncryptionKeySizeCommand: + -- Read Encryption Key Size (v1.1) (BR/EDR) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+2] UInt connection_handle + -- Identifies an active ACL link (only the lower 12 bits are meaningful). + [requires: 0x0000 <= this <= 0x0EFF] + + +struct LESetEventMaskCommand: + -- LE Set Event Mask Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+8] bits: + 0 [+35] LEEventMask le_event_mask + -- Bitmask that indicates which LE events are generated by the HCI for the Host. + + +struct LEReadBufferSizeCommandV1: + -- LE Read Buffer Size Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + +struct LEReadBufferSizeCommandV2: + -- LE Read Buffer Size Command (v5.2) (LE) + -- Version 2 of this command changed the opcode and added ISO return + -- parameters. + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + +struct LEReadLocalSupportedFeaturesCommand: + -- LE Read Local Supported Features Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + +struct LESetRandomAddressCommand: + -- LE Set Random Address Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+BdAddr.$size_in_bytes] BdAddr random_address + + +struct LESetAdvertisingParametersCommand: + -- LE Set Advertising Parameters Command (v4.0) (LE) + + [requires: advertising_interval_min <= advertising_interval_max] + + let hdr_size = CommandHeader.$size_in_bytes + + 0 [+hdr_size] CommandHeader header + + $next [+2] UInt advertising_interval_min + -- Default: 0x0800 (1.28 s) + -- Time: N * 0.625 ms + -- Time Range: 20 ms to 10.24 s + [requires: 0x0020 <= this <= 0x4000] + + $next [+2] UInt advertising_interval_max + -- Default: 0x0800 (1.28 s) + -- Time: N * 0.625 ms + -- Time Range: 20 ms to 10.24 s + [requires: 0x0020 <= this <= 0x4000] + + $next [+1] LEAdvertisingType adv_type + -- Used to determine the packet type that is used for advertising when + -- advertising is enabled. + + $next [+1] LEOwnAddressType own_address_type + + $next [+1] LEPeerAddressType peer_address_type + -- ANONYMOUS address type not allowed. + + $next [+BdAddr.$size_in_bytes] BdAddr peer_address + -- Public Device Address, Random Device Address, Public Identity Address, or + -- Random (static) Identity Address of the device to be connected. + + $next [+1] bits: + + 0 [+3] LEAdvertisingChannels advertising_channel_map + -- Indicates the advertising channels that shall be used when transmitting + -- advertising packets. At least 1 channel must be enabled. + -- Default: all channels enabled + + $next [+1] LEAdvertisingFilterPolicy advertising_filter_policy + -- This parameter shall be ignored when directed advertising is enabled. + + +struct LEReadAdvertisingChannelTxPowerCommand: + -- LE Read Advertising Channel Tx Power Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + +struct LESetAdvertisingDataCommand: + -- LE Set Advertising Data Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+1] UInt advertising_data_length + -- The number of significant octets in `advertising_data`. + [requires: 0x00 <= this <= 0x1F] + + $next [+31] UInt:8[31] advertising_data + -- 31 octets of advertising data formatted as defined in Core Spec + -- v5.3, Vol 3, Part C, Section 11. + -- Default: All octets zero + + +struct LESetScanResponseDataCommand: + -- LE Set Scan Response Data Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+1] UInt scan_response_data_length + -- The number of significant octets in `scan_response_data`. + [requires: 0x00 <= this <= 0x1F] + + $next [+31] UInt:8[31] scan_response_data + -- 31 octets of scan response data formatted as defined in Core Spec + -- v5.3, Vol 3, Part C, Section 11. + -- Default: All octets zero + + +struct LESetScanParametersCommand: + -- LE Set Scan Parameters Command (v4.0) (LE) + + [requires: le_scan_window <= le_scan_interval] + + let hdr_size = CommandHeader.$size_in_bytes + + 0 [+hdr_size] CommandHeader header + + $next [+1] LEScanType le_scan_type + -- Controls the type of scan to perform. + + $next [+2] UInt le_scan_interval + -- Default: 0x0010 (10ms) + -- Time: N * 0.625 ms + -- Time Range: 2.5 ms to 10.24 s + [requires: 0x0004 <= this <= 0x4000] + + $next [+2] UInt le_scan_window + -- Default: 0x0010 (10ms) + -- Time: N * 0.625 ms + -- Time Range: 2.5ms to 10.24 s + [requires: 0x0004 <= this <= 0x4000] + + $next [+1] LEOwnAddressType own_address_type + -- The type of address being used in the scan request packets. + + $next [+1] LEScanFilterPolicy scanning_filter_policy + + +struct LESetScanEnableCommand: + -- LE Set Scan Enable Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+1] GenericEnableParam le_scan_enable + $next [+1] GenericEnableParam filter_duplicates + -- Controls whether the Link Layer should filter out duplicate advertising + -- reports to the Host, or if the Link Layer should generate advertising + -- reports for each packet received. Ignored if le_scan_enable is set to + -- disabled. + -- See Core Spec v5.3, Vol 6, Part B, Section 4.4.3.5 + + +struct LECreateConnectionCommand: + -- LE Create Connection Command (v4.0) (LE) + + [requires: le_scan_window <= le_scan_interval && connection_interval_min <= connection_interval_max] + + let hdr_size = CommandHeader.$size_in_bytes + + 0 [+hdr_size] CommandHeader header + + $next [+2] UInt le_scan_interval + -- The time interval from when the Controller started the last LE scan until + -- it begins the subsequent LE scan. + -- Time: N * 0.625 ms + -- Time Range: 2.5 ms to 10.24 s + [requires: 0x0004 <= this <= 0x4000] + + $next [+2] UInt le_scan_window + -- Amount of time for the duration of the LE scan. + -- Time: N * 0.625 ms + -- Time Range: 2.5 ms to 10.24 s + [requires: 0x0004 <= this <= 0x4000] + + $next [+1] GenericEnableParam initiator_filter_policy + + $next [+1] LEAddressType peer_address_type + + $next [+BdAddr.$size_in_bytes] BdAddr peer_address + + $next [+1] LEOwnAddressType own_address_type + + $next [+2] UInt connection_interval_min + -- Time: N * 1.25 ms + -- Time Range: 7.5 ms to 4 s. + [requires: 0x0006 <= this <= 0x0C80] + + $next [+2] UInt connection_interval_max + -- Time: N * 1.25 ms + -- Time Range: 7.5 ms to 4 s. + [requires: 0x0006 <= this <= 0x0C80] + + $next [+2] UInt max_latency + -- Maximum Peripheral latency for the connection in number of connection + -- events. + [requires: 0x0000 <= this <= 0x01F3] + + $next [+2] UInt supervision_timeout + -- See Core Spec v5.3, Vol 6, Part B, Section 4.5.2. + -- Time: N * 10 ms + -- Time Range: 100 ms to 32 s + [requires: 0x000A <= this <= 0x0C80] + + $next [+2] UInt min_connection_event_length + -- Time: N * 0.625 ms + + $next [+2] UInt max_connection_event_length + -- Time: N * 0.625 ms + + +struct LECreateConnectionCancelCommand: + -- LE Create Connection Cancel Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + +struct LEClearFilterAcceptListCommand: + -- LE Clear Filter Accept List Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + +struct LEAddDeviceToFilterAcceptListCommand: + -- LE Add Device To Filter Accept List Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+1] LEPeerAddressType address_type + -- The address type of the peer. + + $next [+BdAddr.$size_in_bytes] BdAddr address + -- Public Device Address or Random Device Address of the device to be added + -- to the Filter Accept List. Ignored if `address_type` is ANONYMOUS. + + +struct LERemoveDeviceFromFilterAcceptListCommand: + -- LE Remove Device From Filter Accept List Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+1] LEPeerAddressType address_type + -- The address type of the peer. + + $next [+BdAddr.$size_in_bytes] BdAddr address + -- Public Device Address or Random Device Address of the device to be added + -- to the Filter Accept List. Ignored if `address_type` is ANONYMOUS. + + +struct LEConnectionUpdateCommand: + -- LE Connection Update Command (v4.0) (LE) + + [requires: connection_interval_min <= connection_interval_max && min_connection_event_length <= max_connection_event_length] + + let hdr_size = CommandHeader.$size_in_bytes + + 0 [+hdr_size] CommandHeader header + + $next [+2] UInt connection_handle + [requires: 0x0000 <= this <= 0x0EFF] + + $next [+2] UInt connection_interval_min + -- Time: N * 1.25 ms + -- Time Range: 7.5 ms to 4 s. + [requires: 0x0006 <= this <= 0x0C80] + + $next [+2] UInt connection_interval_max + -- Time: N * 1.25 ms + -- Time Range: 7.5 ms to 4 s. + [requires: 0x0006 <= this <= 0x0C80] + + $next [+2] UInt max_latency + -- Maximum Peripheral latency for the connection in number of subrated + -- connection events. + [requires: 0x0000 <= this <= 0x01F3] + + $next [+2] UInt supervision_timeout + -- See Core Spec v5.3, Vol 6, Part B, Section 4.5.2. + -- Time: N * 10 ms + -- Time Range: 100 ms to 32 s + [requires: 0x000A <= this <= 0x0C80] + + $next [+2] UInt min_connection_event_length + -- Time: N * 0.625 ms + + $next [+2] UInt max_connection_event_length + -- Time: N * 0.625 ms + + +struct LEReadRemoteFeaturesCommand: + -- LE Read Remote Features Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+2] UInt connection_handle + [requires: 0x0000 <= this <= 0x0EFF] + + +struct LEEnableEncryptionCommand: + -- LE Enable Encryption Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+2] UInt connection_handle + [requires: 0x0000 <= this <= 0x0EFF] + + $next [+8] UInt random_number + $next [+2] UInt encrypted_diversifier + $next [+LinkKey.$size_in_bytes] LinkKey long_term_key + + +struct LELongTermKeyRequestNegativeReplyCommand: + -- LE Long Term Key Request Negative Reply Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+2] UInt connection_handle + [requires: 0x0000 <= this <= 0x0EFF] + + +struct LEReadSupportedStatesCommand: + -- LE Read Supported States Command (v4.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + +struct LEClearResolvingListCommand: + -- LE Clear Resolving List Command (v4.2) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + + +struct LESetAddressResolutionEnableCommand: + -- LE Set Address Resolution Enable Command (v4.2) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+1] GenericEnableParam address_resolution_enable + + +struct LESetAdvertisingSetRandomAddressCommand: + -- LE Set Advertising Set Random Address Command (v5.0) (LE) + let hdr_size = CommandHeader.$size_in_bytes + 0 [+hdr_size] CommandHeader header + $next [+1] UInt advertising_handle + -- Handle used to identify an advertising set. + + $next [+BdAddr.$size_in_bytes] BdAddr random_address + -- The random address to use in the advertising PDUs. + + +struct WriteAuthenticatedPayloadTimeoutCommand: + -- Write Authenticated Payload Timeout Command (v4.1) (BR/EDR & LE) + 0 [+CommandHeader.$size_in_bytes] CommandHeader header + $next [+2] UInt connection_handle + [requires: 0x0000 <= this <= 0x0EFF] + + $next [+2] UInt authenticated_payload_timeout + -- Default = 0x0BB8 (30 s) + -- Time = N * 10 ms + -- Time Range: 10 ms to 655,350 ms + [requires: 0x0001 <= this <= 0xFFFF] + + +struct ReadAuthenticatedPayloadTimeoutCommand: + -- Read Authenticated Payload Timeout Command (v4.1) (BR/EDR & LE) + 0 [+CommandHeader.$size_in_bytes] CommandHeader header + $next [+2] UInt connection_handle + [requires: 0x0000 <= this <= 0x0EFF] + + +struct ReadLEHostSupportCommand: + -- Read LE Host Support Command (v4.0) (BR/EDR) + 0 [+CommandHeader.$size_in_bytes] CommandHeader header + + +struct ReadFlowControlModeCommand: + -- Read Flow Control Mode Command (v3.0 + HS) (BR/EDR) + 0 [+CommandHeader.$size_in_bytes] CommandHeader header + + +struct WriteFlowControlModeCommand: + -- Write Flow Control Mode Command (v3.0 + HS) (BR/EDR) + 0 [+CommandHeader.$size_in_bytes] CommandHeader header + $next [+1] FlowControlMode flow_control_mode + + +struct SetEventMaskPage2Command: + -- Set Event Mask Page 2 Command (v3.0 + HS) + 0 [+CommandHeader.$size_in_bytes] CommandHeader header + $next [+8] bits: + 0 [+26] EventMaskPage2 event_mask_page_2 + -- Bit mask used to control which HCI events are generated by the HCI for the Host. + # ========================= HCI Event packets =========================== # Core Spec v5.3 Vol 4, Part E, Section 7.7 @@ -1594,6 +2399,160 @@ struct CommandCompleteEvent: let event_fixed_size = $size_in_bytes-hdr_size let return_parameters_size = header.parameter_total_size-event_fixed_size + +struct ConnectionCompleteEvent: + -- Connection Complete Event (v1.1) (BR/EDR) + let hdr_size = EventHeader.$size_in_bytes + 0 [+hdr_size] EventHeader header + $next [+1] StatusCode status + $next [+2] UInt connection_handle + [requires: 0x0000 <= this <= 0x0EFF] + + $next [+BdAddr.$size_in_bytes] BdAddr bd_addr + -- The address of the connected device + + $next [+1] LinkType link_type + $next [+1] GenericEnableParam encryption_enabled + + +struct ConnectionRequestEvent: + -- Connection Request Event (v1.1) (BR/EDR) + let hdr_size = EventHeader.$size_in_bytes + 0 [+hdr_size] EventHeader header + $next [+BdAddr.$size_in_bytes] BdAddr bd_addr + -- The address of the device that's requesting the connection. + + $next [+3] ClassOfDevice class_of_device + -- The Class of Device of the device which requests the connection. + + $next [+1] LinkType link_type + + +struct DisconnectionCompleteEvent: + -- Disconnection Complete Event (v1.1) (BR/EDR & LE) + let hdr_size = EventHeader.$size_in_bytes + 0 [+hdr_size] EventHeader header + $next [+1] StatusCode status + $next [+2] UInt connection_handle + [requires: 0x0000 <= this <= 0x0EFF] + + $next [+1] StatusCode reason + + +struct AuthenticationCompleteEvent: + -- Authentication Complete Event (v1.1) (BR/EDR) + let hdr_size = EventHeader.$size_in_bytes + 0 [+hdr_size] EventHeader header + $next [+1] StatusCode status + $next [+2] UInt connection_handle + [requires: 0x0000 <= this <= 0x0EFF] + + +struct RemoteNameRequestCompleteEvent: + -- Remote Name Request Complete Event (v1.1) (BR/EDR) + let hdr_size = EventHeader.$size_in_bytes + 0 [+hdr_size] EventHeader header + $next [+1] StatusCode status + $next [+BdAddr.$size_in_bytes] BdAddr bd_addr + $next [+248] UInt:8[248] remote_name + -- UTF-8 encoded friendly name. If the name is less than 248 characters, it + -- is null terminated and the remaining bytes are not valid. + + +struct EncryptionChangeEventV1: + -- Encryption Change Event (v1.1) (BR/EDR & LE) + let hdr_size = EventHeader.$size_in_bytes + 0 [+hdr_size] EventHeader header + $next [+1] StatusCode status + $next [+2] UInt connection_handle + [requires: 0x0000 <= this <= 0x0EFF] + + $next [+1] EncryptionStatus encryption_enabled + + +struct ChangeConnectionLinkKeyCompleteEvent: + -- Change Connection Link Key Complete Event (v1.1) (BR/EDR) + let hdr_size = EventHeader.$size_in_bytes + 0 [+hdr_size] EventHeader header + $next [+1] StatusCode status + $next [+2] UInt connection_handle + [requires: 0x0000 <= this <= 0x0EFF] + + +struct ReadRemoteSupportedFeaturesCompleteEvent: + -- Read Remote Supported Features Complete Event (v1.1) (BR/EDR) + let hdr_size = EventHeader.$size_in_bytes + 0 [+hdr_size] EventHeader header + $next [+1] StatusCode status + $next [+2] UInt connection_handle + [requires: 0x0000 <= this <= 0x0EFF] + + $next [+8] LmpFeatures(0) lmp_features + -- Page 0 of the LMP features. + + +struct LEMetaEvent: + let hdr_size = EventHeader.$size_in_bytes + 0 [+hdr_size] EventHeader header + $next [+1] UInt subevent_code + -- The event code for the LE subevent. + + +struct LEConnectionCompleteSubevent: + 0 [+LEMetaEvent.$size_in_bytes] LEMetaEvent le_meta_event + + $next [+1] StatusCode status + + $next [+2] UInt connection_handle + -- Only the lower 12-bits are meaningful. + [requires: 0x0000 <= this <= 0x0EFF] + + $next [+1] ConnectionRole role + + $next [+1] LEPeerAddressType peer_address_type + + $next [+BdAddr.$size_in_bytes] BdAddr peer_address + -- Public Device Address or Random Device Address of the peer device. + + $next [+2] UInt connection_interval + -- Time: N * 1.25 ms + -- Range: 7.5 ms to 4 s + [requires: 0x0006 <= this <= 0x0C80] + + $next [+2] UInt peripheral_latency + [requires: 0x0000 <= this <= 0x01F3] + + $next [+2] UInt supervision_timeout + -- Time: N * 10 ms + -- Range: 100 ms to 32 s + [requires: 0x000A <= this <= 0x0C80] + + $next [+1] LEClockAccuracy central_clock_accuracy + -- Only valid for a peripheral. On a central, this parameter shall be set to 0x00. + + +struct LEConnectionUpdateCompleteSubevent: + 0 [+LEMetaEvent.$size_in_bytes] LEMetaEvent le_meta_event + + $next [+1] StatusCode status + + $next [+2] UInt connection_handle + -- Only the lower 12-bits are meaningful. + [requires: 0x0000 <= this <= 0x0EFF] + + $next [+2] UInt connection_interval + -- Time: N * 1.25 ms + -- Range: 7.5 ms to 4 s + [requires: 0x0006 <= this <= 0x0C80] + + $next [+2] UInt peripheral_latency + [requires: 0x0000 <= this <= 0x01F3] + + $next [+2] UInt supervision_timeout + -- Time: N * 10 ms + -- Range: 100 ms to 32 s + [requires: 0x000A <= this <= 0x0C80] + # ============================ Test packets ============================= diff --git a/pw_bluetooth/public/pw_bluetooth/host.h b/pw_bluetooth/public/pw_bluetooth/host.h index 6fa0ad220..d5f3f3011 100644 --- a/pw_bluetooth/public/pw_bluetooth/host.h +++ b/pw_bluetooth/public/pw_bluetooth/host.h @@ -32,162 +32,162 @@ namespace pw::bluetooth { -// Host is the entrypoint API for interacting with a Bluetooth host stack. Host -// is an abstract class that is implemented by a host stack implementation. +/// Host is the entrypoint API for interacting with a Bluetooth host stack. Host +/// is an abstract class that is implemented by a host stack implementation. class Host { public: - // Represents the persistent configuration of a single Host instance. This is - // used for identity representation in advertisements & bonding secrets - // recall. + /// Represents the persistent configuration of a single Host instance. This is + /// used for identity representation in advertisements & bonding secrets + /// recall. struct PersistentData { - // The local Identity Resolving Key used by a Host to generate Resolvable - // Private Addresses when privacy is enabled. May be absent for hosts that - // do not use LE privacy, or that only use Non-Resolvable Private Addresses. - // - // NOTE: This key is distributed to LE peers during pairing procedures. The - // client must take care to assign an IRK that consistent with the local - // Host identity. + /// The local Identity Resolving Key used by a Host to generate Resolvable + /// Private Addresses when privacy is enabled. May be absent for hosts that + /// do not use LE privacy, or that only use Non-Resolvable Private + /// Addresses. + /// + /// NOTE: This key is distributed to LE peers during pairing procedures. The + /// client must take care to assign an IRK that consistent with the local + /// Host identity. std::optional<Key> identity_resolving_key; - // All bonds that use a public identity address must contain the same local - // address. + /// All bonds that use a public identity address must contain the same local + /// address. span<const low_energy::BondData> bonds; }; - // The security level required for this pairing. This corresponds to the - // security levels defined in the Security Manager Protocol in Core spec v5.3, - // Vol 3, Part H, Section 2.3.1 + /// The security level required for this pairing. This corresponds to the + /// security levels defined in the Security Manager Protocol in Core spec + /// v5.3, Vol 3, Part H, Section 2.3.1 enum class PairingSecurityLevel : uint8_t { - // Encrypted without person-in-the-middle protection (unauthenticated) + /// Encrypted without person-in-the-middle protection (unauthenticated) kEncrypted, - // Encrypted with person-in-the-middle protection (authenticated), although - // this level of security does not fully protect against passive - // eavesdroppers + /// Encrypted with person-in-the-middle protection (authenticated), although + /// this level of security does not fully protect against passive + /// eavesdroppers kAuthenticated, - // Encrypted with person-in-the-middle protection (authenticated). - // This level of security fully protects against eavesdroppers. + /// Encrypted with person-in-the-middle protection (authenticated). + /// This level of security fully protects against eavesdroppers. kLeSecureConnections, }; - // Whether or not the device should form a bluetooth bond during the pairing - // prodecure. As described in Core Spec v5.2, Vol 3, Part C, Sec 4.3 + /// Whether or not the device should form a bluetooth bond during the pairing + /// prodecure. As described in Core Spec v5.2, Vol 3, Part C, Sec 4.3 enum class BondableMode : uint8_t { - // The device will form a bond during pairing with peers + /// The device will form a bond during pairing with peers kBondable, - // The device will not form a bond during pairing with peers + /// The device will not form a bond during pairing with peers kNonBondable, }; - // Parameters that give a caller more fine-grained control over the pairing - // process. + /// Parameters that give a caller more fine-grained control over the pairing + /// process. struct PairingOptions { - // Determines the Security Manager security level to pair with. + /// Determines the Security Manager security level to pair with. PairingSecurityLevel security_level = PairingSecurityLevel::kAuthenticated; - // Indicated whether the device should form a bond or not during pairing. If - // not present, interpreted as bondable mode. + /// Indicated whether the device should form a bond or not during pairing. + /// If not present, interpreted as bondable mode. BondableMode bondable_mode = BondableMode::kBondable; }; - // `Close` should complete before `Host` is destroyed. + /// `Close()` should complete before `Host` is destroyed. virtual ~Host() = default; - // Initializes the host stack. Vendor specific controller initialization (e.g. - // loading firmware) must be done before initializing `Host`. - // - // Parameters: - // `controller` - Pointer to a concrete `Controller` that the host stack - // should use to communicate with the controller. - // `data` - Data to persist from a previous instance of `Host`. - // `on_initialization_complete` - Called when initialization is complete. - // Other methods should not be called until initialization completes. + /// Initializes the host stack. Vendor specific controller initialization + /// (e.g. loading firmware) must be done before initializing `Host`. + /// + /// @param controller Pointer to a concrete `Controller` that the host stack + /// should use to communicate with the controller. + /// @param data Data to persist from a previous instance of `Host`. + /// @param on_initialization_complete Called when initialization is complete. + /// Other methods should not be called until initialization completes. virtual void Initialize( Controller* controller, PersistentData data, Function<void(Status)>&& on_initialization_complete) = 0; - // Safely shuts down the host, ending all active Bluetooth procedures: - // - All objects/pointers associated with this host are destroyed/invalidated - // and all connections disconnected. - // - All scanning and advertising procedures are stopped. - // - // The Host may send events or call callbacks as procedures get terminated. - // `callback` will be called once all procedures have terminated. + /// Safely shuts down the host, ending all active Bluetooth procedures: + /// - All objects/pointers associated with this host are destroyed/invalidated + /// and all connections disconnected. + /// - All scanning and advertising procedures are stopped. + /// + /// The Host may send events or call callbacks as procedures get terminated. + /// @param callback Will be called once all procedures have terminated. virtual void Close(Closure callback) = 0; - // Returns a pointer to the Central API, which is used to scan and connect to - // peers. + /// Returns a pointer to the Central API, which is used to scan and connect to + /// peers. virtual low_energy::Central* Central() = 0; - // Returns a pointer to the Peripheral API, which is used to advertise and - // accept connections from peers. + /// Returns a pointer to the Peripheral API, which is used to advertise and + /// accept connections from peers. virtual low_energy::Peripheral* Peripheral() = 0; - // Returns a pointer to the GATT Server API, which is used to publish GATT - // services. + /// Returns a pointer to the GATT Server API, which is used to publish GATT + /// services. virtual gatt::Server* GattServer() = 0; - // Deletes a peer from the Bluetooth host. If the peer is connected, it will - // be disconnected. `peer_id` will no longer refer to any peer. - // - // Returns `OK` after no peer exists that's identified by `peer_id` (even - // if it didn't exist), `ABORTED` if the peer could not be disconnected or - // deleted and still exists. + /// Deletes a peer from the Bluetooth host. If the peer is connected, it will + /// be disconnected. `peer_id` will no longer refer to any peer. + /// + /// Returns `OK` after no peer exists that's identified by `peer_id` (even + /// if it didn't exist), `ABORTED` if the peer could not be disconnected or + /// deleted and still exists. virtual Status ForgetPeer(PeerId peer_id) = 0; - // Enable or disable the LE privacy feature. When enabled, the host will use a - // private device address in all LE procedures. When disabled, the public - // identity address will be used instead (which is the default). + /// Enable or disable the LE privacy feature. When enabled, the host will use + /// a private device address in all LE procedures. When disabled, the public + /// identity address will be used instead (which is the default). virtual void EnablePrivacy(bool enabled) = 0; - // Set the GAP LE Security Mode of the host. Only encrypted, - // connection-based security modes are supported, i.e. Mode 1 and Secure - // Connections Only mode. If the security mode is set to Secure Connections - // Only, any existing encrypted connections which do not meet the security - // requirements of Secure Connections Only mode will be disconnected. + /// Set the GAP LE Security Mode of the host. Only encrypted, + /// connection-based security modes are supported, i.e. Mode 1 and Secure + /// Connections Only mode. If the security mode is set to Secure Connections + /// Only, any existing encrypted connections which do not meet the security + /// requirements of Secure Connections Only mode will be disconnected. virtual void SetSecurityMode(low_energy::SecurityMode security_mode) = 0; - // Assigns the pairing delegate that will respond to authentication challenges - // using the given I/O capabilities. Calling this method cancels any on-going - // pairing procedure started using a previous delegate. Pairing requests will - // be rejected if no PairingDelegate has been assigned. + /// Assigns the pairing delegate that will respond to authentication + /// challenges using the given I/O capabilities. Calling this method cancels + /// any on-going pairing procedure started using a previous delegate. Pairing + /// requests will be rejected if no PairingDelegate has been assigned. virtual void SetPairingDelegate(InputCapability input, OutputCapability output, PairingDelegate* pairing_delegate) = 0; - // NOTE: This is intended to satisfy test scenarios that require pairing - // procedures to be initiated without relying on service access. In normal - // operation, Bluetooth security is enforced during service access. - // - // Initiates pairing to the peer with the supplied `peer_id` and `options`. - // Returns an error if no connected peer with `peer_id` is found or the - // pairing procedure fails. - // - // If `options` specifies a higher security level than the current pairing, - // this method attempts to raise the security level. Otherwise this method has - // no effect and returns success. - // - // Returns the following errors via `callback`: - // `NOT_FOUND` - The peer `peer_id` was not found. - // `ABORTED` - The pairing procedure failed. + /// NOTE: This is intended to satisfy test scenarios that require pairing + /// procedures to be initiated without relying on service access. In normal + /// operation, Bluetooth security is enforced during service access. + /// + /// Initiates pairing to the peer with the supplied `peer_id` and `options`. + /// Returns an error if no connected peer with `peer_id` is found or the + /// pairing procedure fails. + /// + /// If `options` specifies a higher security level than the current pairing, + /// this method attempts to raise the security level. Otherwise this method + /// has no effect and returns success. + /// + /// Returns the following errors via `callback`: + /// `NOT_FOUND` - The peer `peer_id` was not found. + /// `ABORTED` - The pairing procedure failed. virtual void Pair(PeerId peer_id, PairingOptions options, Function<void(Status)>&& callback) = 0; - // Configures a callback to be called when new bond data for a peer has been - // created. This data should be persisted and used to initialize Host in the - // future. New bond data may be received for an already bonded peer, in which - // case the new data should overwrite the old data. + /// Configures a callback to be called when new bond data for a peer has been + /// created. This data should be persisted and used to initialize Host in the + /// future. New bond data may be received for an already bonded peer, in which + /// case the new data should overwrite the old data. virtual void SetBondDataCallback( Function<void(low_energy::BondData)>&& callback) = 0; - // Looks up the `PeerId` corresponding to `address`. If `address` does not - // correspond to a known peer, a new `PeerId` will be generated for the - // address. If a `PeerId` cannot be generated, std::nullopt will be returned. + /// Looks up the `PeerId` corresponding to `address`. If `address` does not + /// correspond to a known peer, a new `PeerId` will be generated for the + /// address. If a `PeerId` cannot be generated, std::nullopt will be returned. virtual std::optional<PeerId> PeerIdFromAddress(Address address) = 0; - // Looks up the Address corresponding to `peer_id`. Returns null if `peer_id` - // does not correspond to a known peer. + /// Looks up the Address corresponding to `peer_id`. Returns null if `peer_id` + /// does not correspond to a known peer. virtual std::optional<Address> DeviceAddressFromPeerId(PeerId peer_id) = 0; }; diff --git a/pw_bluetooth/public/pw_bluetooth/low_energy/central.h b/pw_bluetooth/public/pw_bluetooth/low_energy/central.h index 2753feb4b..391e691c8 100644 --- a/pw_bluetooth/public/pw_bluetooth/low_energy/central.h +++ b/pw_bluetooth/public/pw_bluetooth/low_energy/central.h @@ -27,104 +27,105 @@ namespace pw::bluetooth::low_energy { -// Represents the LE central role. Used to scan and connect to peripherals. +/// Represents the LE central role. Used to scan and connect to peripherals. class Central { public: - // Represents an ongoing LE scan. - class Scan { + /// Represents an ongoing LE scan. + class ScanHandle { public: + /// Possible errors that can cause a scan to stop prematurely. enum class ScanError : uint8_t { kCanceled = 0 }; - virtual ~Scan() = 0; + virtual ~ScanHandle() = 0; - // Set a callback that will be called if the scan is stopped due to an error - // in the BLE stack. + /// Set a callback that will be called if the scan is stopped due to an + /// error in the BLE stack. virtual void SetErrorCallback(Function<void(ScanError)>&& callback) = 0; private: - // Stop the current scan. This method is called by the ~Scan::Ptr() when it - // goes out of scope, the API client should never call this method. + /// Stop the current scan. This method is called by the ~ScanHandle::Ptr() + /// when it goes out of scope, the API client should never call this method. virtual void StopScan() = 0; public: - // Movable Scan smart pointer. The controller will continue scanning until - // the returned Scan::Ptr is destroyed. - using Ptr = internal::RaiiPtr<Scan, &Scan::StopScan>; + /// Movable ScanHandle smart pointer. The controller will continue scanning + /// until the ScanHandle::Ptr is destroyed. + using Ptr = internal::RaiiPtr<ScanHandle, &ScanHandle::StopScan>; }; - // Filter parameters for use during a scan. A discovered peer only matches the - // filter if it satisfies all of the present filter parameters. + /// Filter parameters for use during a scan. A discovered peer only matches + /// the filter if it satisfies all of the present filter parameters. struct ScanFilter { - // Filter based on advertised service UUID. + /// Filter based on advertised service UUID. std::optional<Uuid> service_uuid; - // Filter based on service data containing the given UUID. + /// Filter based on service data containing the given UUID. std::optional<Uuid> service_data_uuid; - // Filter based on a manufacturer identifier present in the manufacturer - // data. If this filter parameter is set, then the advertising payload must - // contain manufacturer specific data with the provided company identifier - // to satisfy this filter. Manufacturer identifiers can be found at - // https://www.bluetooth.com/specifications/assigned-numbers/company-identifiers/ + /// Filter based on a manufacturer identifier present in the manufacturer + /// data. If this filter parameter is set, then the advertising payload must + /// contain manufacturer specific data with the provided company identifier + /// to satisfy this filter. Manufacturer identifiers can be found at + /// https://www.bluetooth.com/specifications/assigned-numbers/company-identifiers/ std::optional<uint16_t> manufacturer_id; - // Filter based on whether or not a device is connectable. For example, a - // client that is only interested in peripherals that it can connect to can - // set this to true. Similarly a client can scan only for broadcasters by - // setting this to false. + /// Filter based on whether or not a device is connectable. For example, a + /// client that is only interested in peripherals that it can connect to can + /// set this to true. Similarly a client can scan only for broadcasters by + /// setting this to false. std::optional<bool> connectable; - // Filter results based on a portion of the advertised device name. - // Substring matches are allowed. - // The name length must be at most pw::bluetooth::kMaxDeviceNameLength. + /// Filter results based on a portion of the advertised device name. + /// Substring matches are allowed. + /// The name length must be at most pw::bluetooth::kMaxDeviceNameLength. std::optional<std::string_view> name; - // Filter results based on the path loss of the radio wave. A device that - // matches this filter must satisfy the following: - // 1. Radio transmission power level and received signal strength must be - // available for the path loss calculation. - // 2. The calculated path loss value must be less than, or equal to, - // `max_path_loss`. - // - // NOTE: This field is calculated using the RSSI and TX Power information - // obtained from advertising and scan response data during a scan procedure. - // It should NOT be confused with information for an active connection - // obtained using the "Path Loss Reporting" feature. + /// Filter results based on the path loss of the radio wave. A device that + /// matches this filter must satisfy the following: + /// 1. Radio transmission power level and received signal strength must be + /// available for the path loss calculation. + /// 2. The calculated path loss value must be less than, or equal to, + /// `max_path_loss`. + /// + /// @note This field is calculated using the RSSI and TX Power information + /// obtained from advertising and scan response data during a scan + /// procedure. It should NOT be confused with information for an active + /// connection obtained using the "Path Loss Reporting" feature. std::optional<uint8_t> max_path_loss; }; - // Parameters used during a scan. + /// Parameters used during a scan. struct ScanOptions { - // List of filters for use during a scan. A peripheral that satisfies any of - // these filters will be reported. At least 1 filter must be specified. - // While not recommended, clients that require that all peripherals be - // reported can specify an empty filter. + /// List of filters for use during a scan. A peripheral that satisfies any + /// of these filters will be reported. At least 1 filter must be specified. + /// While not recommended, clients that require that all peripherals be + /// reported can specify an empty filter. Vector<ScanFilter> filters; - // The time interval between scans. - // Time = N * 0.625ms - // Range: 0x0004 (2.5ms) - 10.24ms (0x4000) - // Default: 10ms + /// The time interval between scans. + /// - Time = N * 0.625ms + /// - Range: 0x0004 (2.5ms) - 10.24ms (0x4000) + /// - Default: 10ms uint16_t interval = 0x0010; - // The duration of the scan. The window must be less than or equal to the - // interval. - // Time = N * 0.625ms - // Range: 0x0004 (2.5ms) - 10.24ms (0x4000) - // Default: 10ms + /// The duration of the scan. The window must be less than or equal to the + /// interval. + /// - Time = N * 0.625ms + /// - Range: 0x0004 (2.5ms) - 10.24ms (0x4000) + /// - Default: 10ms uint16_t window = 0x0010; }; - // Information obtained from advertising and scan response data broadcast by a - // peer. + /// Information obtained from advertising and scan response data broadcast by + /// a peer. struct ScanData { - // The radio transmit power level. - // NOTE: This field should NOT be confused with the "connection TX Power - // Level" of a peer that is currently connected to the system obtained via - // the "Transmit Power reporting" feature. + /// The radio transmit power level. + /// @note This field should NOT be confused with the "connection TX Power + /// Level" of a peer that is currently connected to the system obtained via + /// the "Transmit Power reporting" feature. std::optional<uint8_t> tx_power; - // The appearance of the device. + /// The appearance of the device. std::optional<Appearance> appearance; Vector<Uuid> service_uuids; @@ -133,122 +134,121 @@ class Central { Vector<ManufacturerData> manufacturer_data; - // String representing a URI to be advertised, as defined in IETF STD - // 66: https://tools.ietf.org/html/std66. Each entry should be a UTF-8 - // string including the scheme. For more information, see - // https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml for - // allowed schemes; NOTE: Bluetooth advertising compresses schemas over the - // air to save space. See - // https://www.bluetooth.com/specifications/assigned-numbers/uri-scheme-name-string-mapping. + /// String representing a URI to be advertised, as defined in IETF STD 66: + /// https://tools.ietf.org/html/std66. Each entry should be a UTF-8 string + /// including the scheme. For more information, see + /// https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml for + /// allowed schemes; + /// @note Bluetooth advertising compresses schemas over the air to save + /// space. See + /// https://www.bluetooth.com/specifications/assigned-numbers/uri-scheme-name-string-mapping. Vector<std::string_view> uris; - // The time when this scan data was received. + /// The time when this scan data was received. chrono::SystemClock::time_point timestamp; }; struct ScanResult { - // ScanResult is non-copyable becuase strings are only valid in the - // result callback. + /// ScanResult is non-copyable because strings are only valid in the result + /// callback. ScanResult(const ScanResult&) = delete; ScanResult& operator=(const ScanResult&) = delete; - // Uniquely identifies this peer on the current system. + /// Uniquely identifies this peer on the current system. PeerId peer_id; - // Whether or not this peer is connectable. Non-connectable peers are - // typically in the LE broadcaster role. + /// Whether or not this peer is connectable. Non-connectable peers are + /// typically in the LE broadcaster role. bool connectable; - // The last observed signal strength of this peer. This field is only - // present for a peer that is broadcasting. The RSSI can be stale if the - // peer has not been advertising. - // - // NOTE: This field should NOT be confused with the "connection RSSI" of a - // peer that is currently connected to the system. + /// The last observed signal strength of this peer. This field is only + /// present for a peer that is broadcasting. The RSSI can be stale if the + /// peer has not been advertising. + /// + /// @note This field should NOT be confused with the "connection RSSI" of a + /// peer that is currently connected to the system. std::optional<uint8_t> rssi; - // Information from advertising and scan response data broadcast by this - // peer. This contains the advertising data last received from the peer. + /// Information from advertising and scan response data broadcast by this + /// peer. This contains the advertising data last received from the peer. ScanData scan_data; - // The name of this peer. The name is often obtained during a scan procedure - // and can get updated during the name discovery procedure following a - // connection. - // - // This field is present if the name is known. + /// The name of this peer. The name is often obtained during a scan + /// procedure and can get updated during the name discovery procedure + /// following a connection. + /// + /// This field is present if the name is known. std::optional<std::string_view> name; - // Timestamp of when the information in this `ScanResult` was last updated. + /// Timestamp of when the information in this `ScanResult` was last updated. chrono::SystemClock::time_point last_updated; }; - // Possible errors returned by `Connect`. + /// Possible errors returned by `Connect`. enum class ConnectError : uint8_t { - // The peer ID is unknown. + /// The peer ID is unknown. kUnknownPeer, - // The `ConnectionOptions` were invalid. + /// The `ConnectionOptions` were invalid. kInvalidOptions, - // A connection to the peer already exists. + /// A connection to the peer already exists. kAlreadyExists, - // A connection could not be established. + /// A connection could not be established. kCouldNotBeEstablished, }; enum class StartScanError : uint8_t { - // A scan is already in progress. Only 1 scan may be active at a time. + /// A scan is already in progress. Only 1 scan may be active at a time. kScanInProgress, - // Some of the scan options are invalid. + /// Some of the scan options are invalid. kInvalidParameters, - // An internal error occurred and a scan could not be started. + /// An internal error occurred and a scan could not be started. kInternal, }; - // The Result type returned by Connect() via the passed callback. + /// The Result type returned by Connect() via the passed callback. using ConnectResult = Result<ConnectError, Connection::Ptr>; virtual ~Central() = default; - // Connect to the peer with the given identifier. - // - // The requested `Connection` represents the client's interest in the LE - // connection to the peer. Destroying the `Connection` will disconnect from - // the peer. Only 1 connection per peer may exist at a time. - // - // The `Connection` will be closed by the system if the connection to the peer - // is lost or an error occurs, as indicated by `Connection.OnError`. - // - // Parameters: - // `id` - Identifier of the peer to initiate a connection to. - // `options` - Options used to configure the connection. - // `callback` - Called when a connection is successfully established, or an - // error occurs. - // - // Possible errors are documented in `ConnectError`. + /// Connect to the peer with the given identifier. + /// + /// The requested `Connection` represents the client's interest in the LE + /// connection to the peer. Destroying the `Connection` will disconnect from + /// the peer. Only 1 connection per peer may exist at a time. + /// + /// The `Connection` will be closed by the system if the connection to the + /// peer is lost or an error occurs, as indicated by `Connection.OnError`. + /// + /// @param peer_id Identifier of the peer to initiate a connection to. + /// @param options Options used to configure the connection. + /// @param callback Called when a connection is successfully established, or + /// an error occurs. + /// + /// Possible errors are documented in `ConnectError`. virtual void Connect(PeerId peer_id, ConnectionOptions options, Function<void(ConnectResult)>&& callback) = 0; - // Scans for nearby LE peripherals and broadcasters. The lifetime of the scan - // session is tied to the returned `Scan` object. Once a scan is started, - // `scan_result_callback` will be called with scan results. Only 1 scan may be - // active at a time. It is OK to destroy the `Scan::Ptr` object in - // `scan_result_callback` to stop scanning (no more results will be returned). - // - // Parameters: - // `options` - Options used to configure the scan session. - // `scan_result_callback` - If scanning starts successfully,called for LE - // peers that satisfy the filters indicated in `options`. The initial - // calls may report recently discovered peers. Subsequent calls will - // be made only when peers have been scanned or updated since the last - // call. - // `scan_started_callback` - Called with a `Scan` object if the - // scan successfully starts, or a `ScanError` otherwise. + /// Scans for nearby LE peripherals and broadcasters. The lifetime of the scan + /// session is tied to the returned `ScanHandle` object. Once a scan is + /// started, `scan_result_callback` will be called with scan results. Only 1 + /// scan may be active at a time. It is OK to destroy the `ScanHandle::Ptr` + /// object in `scan_result_callback` to stop scanning (no more results will be + /// returned). + /// + /// @param options Options used to configure the scan session. + /// @param scan_result_callback If scanning starts successfully,called for LE + /// peers that satisfy the filters indicated in `options`. The initial calls + /// may report recently discovered peers. Subsequent calls will be made only + /// when peers have been scanned or updated since the last call. + /// @param scan_started_callback Called with a `ScanHandle` object if the scan + /// successfully starts, or a `ScanError` otherwise. virtual void Scan(ScanOptions options, Function<void(ScanResult)>&& scan_result_callback, - Function<void(Result<StartScanError, Scan::Ptr>)>&& + Function<void(Result<StartScanError, ScanHandle::Ptr>)>&& scan_started_callback) = 0; }; diff --git a/pw_bluetooth/public/pw_bluetooth/low_energy/connection.h b/pw_bluetooth/public/pw_bluetooth/low_energy/connection.h index e7d823faa..d3109535d 100644 --- a/pw_bluetooth/public/pw_bluetooth/low_energy/connection.h +++ b/pw_bluetooth/public/pw_bluetooth/low_energy/connection.h @@ -19,84 +19,84 @@ namespace pw::bluetooth::low_energy { -// Actual connection parameters returned by the controller. +/// Actual connection parameters returned by the controller. struct ConnectionParameters { - // The connection interval indicates the frequency of link layer connection - // events over which data channel PDUs can be transmitted. See Core Spec v5.3, - // Vol 6, Part B, Section 4.5.1 for more information on the link layer - // connection events. - // Range: 0x0006 to 0x0C80 - // Time: N * 1.25 ms - // Time Range: 7.5 ms to 4 s. + /// The connection interval indicates the frequency of link layer connection + /// events over which data channel PDUs can be transmitted. See Core Spec + /// v5.3, Vol 6, Part B, Section 4.5.1 for more information on the link layer + /// connection events. + /// - Range: 0x0006 to 0x0C80 + /// - Time: N * 1.25 ms + /// - Time Range: 7.5 ms to 4 s. uint16_t interval; - // The maximum allowed peripheral connection latency in number of connection - // events. See Core Spec v5.3, Vol 6, Part B, Section 4.5.2. - // Range: 0x0000 to 0x01F3 + /// The maximum allowed peripheral connection latency in number of connection + /// events. See Core Spec v5.3, Vol 6, Part B, Section 4.5.2. + /// - Range: 0x0000 to 0x01F3 uint16_t latency; - // This defines the maximum time between two received data packet PDUs - // before the connection is considered lost. See Core Spec v5.3, Vol 6, Part - // B, Section 4.5.2. - // Range: 0x000A to 0x0C80 - // Time: N * 10 ms - // Time Range: 100 ms to 32 s + /// This defines the maximum time between two received data packet PDUs + /// before the connection is considered lost. See Core Spec v5.3, Vol 6, Part + /// B, Section 4.5.2. + /// - Range: 0x000A to 0x0C80 + /// - Time: N * 10 ms + /// - Time Range: 100 ms to 32 s uint16_t supervision_timeout; }; -// Connection parameters that either the local device or a peer device are -// requesting. +/// Connection parameters that either the local device or a peer device are +/// requesting. struct RequestedConnectionParameters { - // Minimum value for the connection interval. This shall be less than or equal - // to `max_interval`. The connection interval indicates the frequency of link - // layer connection events over which data channel PDUs can be transmitted. - // See Core Spec v5.3, Vol 6, Part B, Section 4.5.1 for more information on - // the link layer connection events. - // Range: 0x0006 to 0x0C80 - // Time: N * 1.25 ms - // Time Range: 7.5 ms to 4 s. + /// Minimum value for the connection interval. This shall be less than or + /// equal to `max_interval`. The connection interval indicates the frequency + /// of link layer connection events over which data channel PDUs can be + /// transmitted. See Core Spec v5.3, Vol 6, Part B, Section 4.5.1 for more + /// information on the link layer connection events. + /// - Range: 0x0006 to 0x0C80 + /// - Time: N * 1.25 ms + /// - Time Range: 7.5 ms to 4 s. uint16_t min_interval; - // Maximum value for the connection interval. This shall be greater than or - // equal to `min_interval`. The connection interval indicates the frequency - // of link layer connection events over which data channel PDUs can be - // transmitted. See Core Spec v5.3, Vol 6, Part B, Section 4.5.1 for more - // information on the link layer connection events. - // Range: 0x0006 to 0x0C80 - // Time: N * 1.25 ms - // Time Range: 7.5 ms to 4 s. + /// Maximum value for the connection interval. This shall be greater than or + /// equal to `min_interval`. The connection interval indicates the frequency + /// of link layer connection events over which data channel PDUs can be + /// transmitted. See Core Spec v5.3, Vol 6, Part B, Section 4.5.1 for more + /// information on the link layer connection events. + /// - Range: 0x0006 to 0x0C80 + /// - Time: N * 1.25 ms + /// - Time Range: 7.5 ms to 4 s. uint16_t max_interval; - // Maximum peripheral latency for the connection in number of connection - // events. See Core Spec v5.3, Vol 6, Part B, Section 4.5.2. - // Range: 0x0000 to 0x01F3 + /// Maximum peripheral latency for the connection in number of connection + /// events. See Core Spec v5.3, Vol 6, Part B, Section 4.5.2. + /// - Range: 0x0000 to 0x01F3 uint16_t max_latency; - // This defines the maximum time between two received data packet PDUs - // before the connection is considered lost. See Core Spec v5.3, Vol 6, Part - // B, Section 4.5.2. - // Range: 0x000A to 0x0C80 - // Time: N * 10 ms - // Time Range: 100 ms to 32 s + /// This defines the maximum time between two received data packet PDUs + /// before the connection is considered lost. See Core Spec v5.3, Vol 6, Part + /// B, Section 4.5.2. + /// - Range: 0x000A to 0x0C80 + /// - Time: N * 10 ms + /// - Time Range: 100 ms to 32 s uint16_t supervision_timeout; }; -// Represents parameters that are set on a per-connection basis. +/// Represents parameters that are set on a per-connection basis. struct ConnectionOptions { - // When true, the connection operates in bondable mode. This means pairing - // will form a bond, or persist across disconnections, if the peer is also - // in bondable mode. When false, the connection operates in non-bondable - // mode, which means the local device only allows pairing that does not form - // a bond. + /// When true, the connection operates in bondable mode. This means pairing + /// will form a bond, or persist across disconnections, if the peer is also + /// in bondable mode. When false, the connection operates in non-bondable + /// mode, which means the local device only allows pairing that does not form + /// a bond. bool bondable_mode = true; - // When present, service discovery performed following the connection is - // restricted to primary services that match this field. Otherwise, by - // default all available services are discovered. + /// When present, service discovery performed following the connection is + /// restricted to primary services that match this field. Otherwise, by + /// default all available services are discovered. std::optional<Uuid> service_filter; - // When present, specifies the initial connection parameters. Otherwise, the - // connection parameters will be selected by the implementation. + /// When present, specifies the initial connection parameters. Otherwise, the + /// connection parameters will be selected by the implementation. std::optional<RequestedConnectionParameters> parameters; }; @@ -107,66 +107,66 @@ struct ConnectionOptions { /// represents. Destroying the object results in a disconnection. class Connection { public: - // Possible errors when updating the connection parameters. + /// Possible errors when updating the connection parameters. enum class ConnectionParameterUpdateError : uint8_t { kFailure, kInvalidParameters, kRejected, }; - // Possible reasons a connection was disconnected. + /// Possible reasons a connection was disconnected. enum class DisconnectReason : uint8_t { kFailure, kRemoteUserTerminatedConnection, - // This usually indicates that the link supervision timeout expired. + /// This usually indicates that the link supervision timeout expired. kConnectionTimeout, }; - // If a disconnection has not occurred, destroying this object will result in - // disconnection. + /// If a disconnection has not occurred, destroying this object will result in + /// disconnection. virtual ~Connection() = default; - // Sets a callback that will be called when the peer disconnects or there is a - // connection error that causes a disconnection. This should be configured by - // the client immediately after establishing the connection. `callback` will - // not be called for disconnections initiated by the client (e.g. by - // destroying `Connection`). It is OK to destroy this object from within - // `callback`. + /// Sets a callback that will be called when the peer disconnects or there is + /// a connection error that causes a disconnection. This should be configured + /// by the client immediately after establishing the connection. `callback` + /// will not be called for disconnections initiated by the client (e.g. by + /// destroying `Connection`). It is OK to destroy this object from within + /// `callback`. virtual void SetDisconnectCallback( Function<void(DisconnectReason)>&& callback) = 0; - // Returns a GATT client to the connected peer that is valid for the lifetime - // of this connection. The client is valid for the lifetime of this - // connection. + /// Returns a GATT client to the connected peer that is valid for the lifetime + /// of this connection. The client is valid for the lifetime of this + /// connection. virtual gatt::Client* GattClient() = 0; - // Returns the current ATT Maximum Transmission Unit. By subtracting ATT - // headers from the MTU, the maximum payload size of messages can be - // calculated. + /// Returns the current ATT Maximum Transmission Unit. By subtracting ATT + /// headers from the MTU, the maximum payload size of messages can be + /// calculated. virtual uint16_t AttMtu() = 0; - // Sets a callback that will be called with the new ATT MTU whenever it is - // updated. + /// Sets a callback that will be called with the new ATT MTU whenever it is + /// updated. virtual void SetAttMtuChangeCallback(Function<void(uint16_t)> callback) = 0; - // Returns the current connection parameters. + /// Returns the current connection parameters. virtual ConnectionParameters Parameters() = 0; - // Requests an update to the connection parameters. `callback` will be called - // with the result of the request. + /// Requests an update to the connection parameters. `callback` will be called + /// with the result of the request. virtual void RequestConnectionParameterUpdate( RequestedConnectionParameters parameters, Function<void(Result<ConnectionParameterUpdateError>)>&& callback) = 0; private: - // Request to disconnect this connection. This method is called by the - // ~Connection::Ptr() when it goes out of scope, the API client should never - // call this method. + /// Request to disconnect this connection. This method is called by the + /// ~Connection::Ptr() when it goes out of scope, the API client should never + /// call this method. virtual void Disconnect() = 0; public: - // Movable Connection smart pointer. When Connection::Ptr is destroyed the - // Connection will disconnect automatically. + /// Movable Connection smart pointer. When Connection::Ptr is destroyed the + /// Connection will disconnect automatically. using Ptr = internal::RaiiPtr<Connection, &Connection::Disconnect>; }; diff --git a/pw_bluetooth/public/pw_bluetooth/low_energy/peripheral.h b/pw_bluetooth/public/pw_bluetooth/low_energy/peripheral.h index 23c56d170..34c88fc1e 100644 --- a/pw_bluetooth/public/pw_bluetooth/low_energy/peripheral.h +++ b/pw_bluetooth/public/pw_bluetooth/low_energy/peripheral.h @@ -26,130 +26,133 @@ namespace pw::bluetooth::low_energy { -// `AdvertisedPeripheral` instances are valid for the duration of advertising. +/// `AdvertisedPeripheral` instances are valid for the duration of advertising. class AdvertisedPeripheral { public: virtual ~AdvertisedPeripheral() = default; - // Set a callback that will be called when an error occurs and advertising - // has been stopped (invalidating this object). It is OK to destroy the - // `AdvertisedPeripheral::Ptr` object from within `callback`. + /// Set a callback that will be called when an error occurs and advertising + /// has been stopped (invalidating this object). It is OK to destroy the + /// `AdvertisedPeripheral::Ptr` object from within `callback`. virtual void SetErrorCallback(Closure callback) = 0; - // For connectable advertisements, this callback will be called when an LE - // central connects to the advertisement. It is recommended to set this - // callback immediately after advertising starts to avoid dropping - // connections. - // - // The returned Connection can be used to interact with the peer. It also - // represents a peripheral's ownership over the connection: the client can - // drop the object to request a disconnection. Similarly, the Connection - // error handler is called by the system to indicate that the connection to - // the peer has been lost. While connections are exclusive among peripherals, - // they may be shared with centrals. - // - // If advertising is not stopped, this callback may be called with multiple - // connections over the lifetime of an advertisement. It is OK to destroy - // the `AdvertisedPeripheral::Ptr` object from within `callback` in order to - // stop advertising. + /// For connectable advertisements, this callback will be called when an LE + /// central connects to the advertisement. It is recommended to set this + /// callback immediately after advertising starts to avoid dropping + /// connections. + /// + /// The returned Connection can be used to interact with the peer. It also + /// represents a peripheral's ownership over the connection: the client can + /// drop the object to request a disconnection. Similarly, the Connection + /// error handler is called by the system to indicate that the connection to + /// the peer has been lost. While connections are exclusive among peripherals, + /// they may be shared with centrals. + /// + /// If advertising is not stopped, this callback may be called with multiple + /// connections over the lifetime of an advertisement. It is OK to destroy + /// the `AdvertisedPeripheral::Ptr` object from within `callback` in order to + /// stop advertising. virtual void SetConnectionCallback( Function<void(Connection::Ptr)>&& callback) = 0; private: - // Stop advertising. This method is called by the ~AdvertisedPeripheral::Ptr() - // when it goes out of scope, the API client should never call this method. + /// Stop advertising. This method is called by the + /// ~AdvertisedPeripheral::Ptr() when it goes out of scope, the API client + /// should never call this method. virtual void StopAdvertising() = 0; public: - // Movable AdvertisedPeripheral smart pointer. The peripheral will continue - // advertising until the returned AdvertisedPeripheral::Ptr is destroyed. + /// Movable AdvertisedPeripheral smart pointer. The peripheral will continue + /// advertising until the returned AdvertisedPeripheral::Ptr is destroyed. using Ptr = internal::RaiiPtr<AdvertisedPeripheral, &AdvertisedPeripheral::StopAdvertising>; }; -// Represents the LE Peripheral role, which advertises and is connected to. +/// Represents the LE Peripheral role, which advertises and is connected to. class Peripheral { public: - // The range of the time interval between advertisements. Shorter intervals - // result in faster discovery at the cost of higher power consumption. The - // exact interval used is determined by the Bluetooth controller. - // Time = N * 0.625ms. - // Time range: 0x0020 (20ms) - 0x4000 (10.24s) + /// The range of the time interval between advertisements. Shorter intervals + /// result in faster discovery at the cost of higher power consumption. The + /// exact interval used is determined by the Bluetooth controller. + /// - Time = N * 0.625ms. + /// - Time range: 0x0020 (20ms) - 0x4000 (10.24s) struct AdvertisingIntervalRange { - uint16_t min = 0x0800; // 1.28s - uint16_t max = 0x0800; // 1.28s + /// Default: 1.28s + uint16_t min = 0x0800; + /// Default: 1.28s + uint16_t max = 0x0800; }; - // Represents the parameters for configuring advertisements. + /// Represents the parameters for configuring advertisements. struct AdvertisingParameters { - // The fields that will be encoded in the data section of advertising - // packets. + /// The fields that will be encoded in the data section of advertising + /// packets. AdvertisingData data; - // The fields that are to be sent in a scan response packet. Clients may use - // this to send additional data that does not fit inside an advertising - // packet on platforms that do not support the advertising data length - // extensions. - // - // If present advertisements will be configured to be scannable. + /// The fields that are to be sent in a scan response packet. Clients may + /// use this to send additional data that does not fit inside an advertising + /// packet on platforms that do not support the advertising data length + /// extensions. + /// + /// If present advertisements will be configured to be scannable. std::optional<AdvertisingData> scan_response; - // See `AdvertisingIntervalRange` documentation. + /// See `AdvertisingIntervalRange` documentation. AdvertisingIntervalRange interval_range; - // If present, the controller will broadcast connectable advertisements - // which allow peers to initiate connections to the Peripheral. The fields - // of `ConnectionOptions` will configure any connections set up from - // advertising. + /// If present, the controller will broadcast connectable advertisements + /// which allow peers to initiate connections to the Peripheral. The fields + /// of `ConnectionOptions` will configure any connections set up from + /// advertising. std::optional<ConnectionOptions> connection_options; }; - // Errors returned by `Advertise`. + /// Errors returned by `Advertise`. enum class AdvertiseError { - // The operation or parameters requested are not supported on the current - // hardware. + /// The operation or parameters requested are not supported on the current + /// hardware. kNotSupported = 1, - // The provided advertising data exceeds the maximum allowed length when - // encoded. + /// The provided advertising data exceeds the maximum allowed length when + /// encoded. kAdvertisingDataTooLong = 2, - // The provided scan response data exceeds the maximum allowed length when - // encoded. + /// The provided scan response data exceeds the maximum allowed length when + /// encoded. kScanResponseDataTooLong = 3, - // The requested parameters are invalid. + /// The requested parameters are invalid. kInvalidParameters = 4, - // This may be called if the maximum number of simultaneous advertisements - // has already been reached. + /// This may be called if the maximum number of simultaneous advertisements + /// has already been reached. kNotEnoughAdvertisingSlots = 5, - // Advertising could not be initiated due to a hardware or system error. + /// Advertising could not be initiated due to a hardware or system error. kFailed = 6, }; + /// Callback for `Advertise()` method. using AdvertiseCallback = Function<void(Result<AdvertiseError, AdvertisedPeripheral::Ptr>)>; virtual ~Peripheral() = default; - // Start advertising continuously as a LE peripheral. If advertising cannot - // be initiated then `result_callback` will be called with an error. Once - // started, advertising can be stopped by destroying the returned - // `AdvertisedPeripheral::Ptr`. - // - // If the system supports multiple advertising, this may be called as many - // times as there are advertising slots. To reconfigure an advertisement, - // first close the original advertisement and then initiate a new - // advertisement. - // - // Parameters: - // `parameters` - Parameters used while configuring the advertising - // instance. - // `result_callback` - Called once advertising has started or failed. On - // success, called with an `AdvertisedPeripheral` that models the lifetime - // of the advertisement. Destroying it will stop advertising. + /// Start advertising continuously as a LE peripheral. If advertising cannot + /// be initiated then `result_callback` will be called with an error. Once + /// started, advertising can be stopped by destroying the returned + /// `AdvertisedPeripheral::Ptr`. + /// + /// If the system supports multiple advertising, this may be called as many + /// times as there are advertising slots. To reconfigure an advertisement, + /// first close the original advertisement and then initiate a new + /// advertisement. + /// + /// @param parameters Parameters used while configuring the advertising + /// instance. + /// @param result_callback Called once advertising has started or failed. On + /// success, called with an `AdvertisedPeripheral` that models the lifetime of + /// the advertisement. Destroying it will stop advertising. virtual void Advertise(const AdvertisingParameters& parameters, AdvertiseCallback&& result_callback) = 0; }; diff --git a/pw_bluetooth/public/pw_bluetooth/types.h b/pw_bluetooth/public/pw_bluetooth/types.h index 3639bbf71..d969b6280 100644 --- a/pw_bluetooth/public/pw_bluetooth/types.h +++ b/pw_bluetooth/public/pw_bluetooth/types.h @@ -98,4 +98,4 @@ enum class Appearance : uint16_t { kSportsActivityLocationAndNavPod = 5188, }; -} // namespace pw::bluetooth
\ No newline at end of file +} // namespace pw::bluetooth diff --git a/pw_bluetooth/public/pw_bluetooth/vendor.emb b/pw_bluetooth/public/pw_bluetooth/vendor.emb index 2c5681bf6..9ee77343a 100644 --- a/pw_bluetooth/public/pw_bluetooth/vendor.emb +++ b/pw_bluetooth/public/pw_bluetooth/vendor.emb @@ -27,6 +27,17 @@ import "hci.emb" as hci # ======================= Android HCI extensions ======================== # Documentation: https://source.android.com/devices/bluetooth/hci_requirements +enum Capability: + [maximum_bits: 8] + NOT_CAPABLE = 0x00 + CAPABLE = 0x01 + +bits AudioCodecSupportMask: + 0 [+1] Flag sbc + 1 [+1] Flag aac + 2 [+1] Flag aptx + 3 [+1] Flag aptx_hd + 4 [+1] Flag ldac # ============ Commands ============ @@ -39,6 +50,10 @@ struct LEMultiAdvtEnableCommand: $next [+1] hci.GenericEnableParam enable $next [+1] UInt advertising_handle +struct LEGetVendorCapabilitiesCommand: + let hdr_size = hci.CommandHeader.$size_in_bytes + 0 [+hdr_size] hci.CommandHeader header + # ============ Events ============ @@ -56,3 +71,42 @@ struct LEMultiAdvtStateChangeSubevent: $next [+2] UInt connection_handle -- Handle used to identify the connection that caused the state change (i.e. -- advertising instance to be disabled). Value will be 0xFFFF if invalid. + +struct LEGetVendorCapabilitiesCommandCompleteEvent: + let hdr_size = hci.CommandCompleteEvent.$size_in_bytes + 0 [+hdr_size] hci.CommandCompleteEvent command_complete + $next [+1] hci.StatusCode status + $next [+1] UInt max_advt_instances + -- Number of advertisement instances supported + -- Deprecated in Google feature spec v0.98 and higher + $next [+1] Capability offloaded_resolution_of_private_address + -- BT chip capability of RPA + -- Deprecated in Google feature spec v0.98 and higher + $next [+2] UInt total_scan_results_storage + -- Storage for scan results in bytes + $next [+1] UInt max_irk_list_sz + -- Number of IRK entries supported in the firmware + $next [+1] Capability filtering_support + -- Support for filtering in the controller + $next [+1] UInt max_filter + -- Number of filters supported + $next [+1] Capability activity_energy_info_support + -- Supports reporting of activity and energy information + $next [+2] bits version_supported: + -- Specifies the version of the Google feature spec supported + 0 [+8] UInt major_number + $next [+8] UInt minor_number + $next [+2] UInt total_num_of_advt_tracked + -- Total number of advertisers tracked for OnLost/OnFound purposes + $next [+1] Capability extended_scan_support + -- Supports extended scan window and interval + $next [+1] Capability debug_logging_supported + -- Supports logging of binary debug information from controller + $next [+1] Capability le_address_generation_offloading_support + -- Deprecated in Google feature spec v0.98 and higher + $next [+4] bits: + 0 [+5] AudioCodecSupportMask a2dp_source_offload_capability_mask + $next [+1] Capability bluetooth_quality_report_support + -- Supports reporting of Bluetooth Quality events + $next [+4] bits: + 0 [+5] AudioCodecSupportMask dynamic_audio_buffer_support diff --git a/pw_bluetooth_hci/BUILD.gn b/pw_bluetooth_hci/BUILD.gn index e4082fe63..0f5107163 100644 --- a/pw_bluetooth_hci/BUILD.gn +++ b/pw_bluetooth_hci/BUILD.gn @@ -63,10 +63,14 @@ pw_test_group("tests") { tests = [ ":packet_test", ":uart_transport_test", - ":uart_transport_fuzzer", + ":uart_transport_fuzzer_test", ] } +group("fuzzers") { + deps = [ ":uart_transport_fuzzer" ] +} + pw_test("packet_test") { sources = [ "packet_test.cc" ] deps = [ diff --git a/pw_bluetooth_hci/uart_transport_fuzzer.cc b/pw_bluetooth_hci/uart_transport_fuzzer.cc index a2f1249c8..4324c4ff9 100644 --- a/pw_bluetooth_hci/uart_transport_fuzzer.cc +++ b/pw_bluetooth_hci/uart_transport_fuzzer.cc @@ -36,16 +36,18 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { const CommandPacket& command_packet = packet.command_packet(); const uint16_t opcode = command_packet.opcode(); - stream.Write(as_bytes(span<const uint16_t>(&opcode, 1))); + stream.Write(as_bytes(span<const uint16_t>(&opcode, 1))).IgnoreError(); const uint16_t opcode_command_field = command_packet.opcode_command_field(); - stream.Write(as_bytes(span<const uint16_t>(&opcode_command_field, 1))); + stream.Write(as_bytes(span<const uint16_t>(&opcode_command_field, 1))) + .IgnoreError(); const uint8_t opcode_group_field = command_packet.opcode_group_field(); - stream.Write(as_bytes(span<const uint8_t>(&opcode_group_field, 1))); + stream.Write(as_bytes(span<const uint8_t>(&opcode_group_field, 1))) + .IgnoreError(); - stream.Write(command_packet.parameters()); + stream.Write(command_packet.parameters()).IgnoreError(); return; } @@ -54,19 +56,21 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { const uint16_t handle_and_fragmentation_bits = async_data_packet.handle_and_fragmentation_bits(); - stream.Write( - as_bytes(span<const uint16_t>(&handle_and_fragmentation_bits, 1))); + stream + .Write(as_bytes( + span<const uint16_t>(&handle_and_fragmentation_bits, 1))) + .IgnoreError(); const uint16_t handle = async_data_packet.handle(); - stream.Write(as_bytes(span<const uint16_t>(&handle, 1))); + stream.Write(as_bytes(span<const uint16_t>(&handle, 1))).IgnoreError(); const uint8_t pb_flag = async_data_packet.pb_flag(); - stream.Write(as_bytes(span<const uint8_t>(&pb_flag, 1))); + stream.Write(as_bytes(span<const uint8_t>(&pb_flag, 1))).IgnoreError(); const uint8_t bc_flag = async_data_packet.bc_flag(); - stream.Write(as_bytes(span<const uint8_t>(&bc_flag, 1))); + stream.Write(as_bytes(span<const uint8_t>(&bc_flag, 1))).IgnoreError(); - stream.Write(async_data_packet.data()); + stream.Write(async_data_packet.data()).IgnoreError(); return; } @@ -75,17 +79,18 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { const uint16_t handle_and_status_bits = sync_data_packet.handle_and_status_bits(); - stream.Write( - as_bytes(span<const uint16_t>(&handle_and_status_bits, 1))); + stream.Write(as_bytes(span<const uint16_t>(&handle_and_status_bits, 1))) + .IgnoreError(); const uint16_t handle = sync_data_packet.handle(); - stream.Write(as_bytes(span<const uint16_t>(&handle, 1))); + stream.Write(as_bytes(span<const uint16_t>(&handle, 1))).IgnoreError(); const uint8_t packet_status_flag = sync_data_packet.packet_status_flag(); - stream.Write(as_bytes(span<const uint8_t>(&packet_status_flag, 1))); + stream.Write(as_bytes(span<const uint8_t>(&packet_status_flag, 1))) + .IgnoreError(); - stream.Write(sync_data_packet.data()); + stream.Write(sync_data_packet.data()).IgnoreError(); return; } @@ -93,9 +98,10 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { const EventPacket& event_packet = packet.event_packet(); const uint8_t event_code = event_packet.event_code(); - stream.Write(as_bytes(span<const uint8_t>(&event_code, 1))); + stream.Write(as_bytes(span<const uint8_t>(&event_code, 1))) + .IgnoreError(); - stream.Write(event_packet.parameters()); + stream.Write(event_packet.parameters()).IgnoreError(); return; } diff --git a/pw_bluetooth_profiles/device_info_service_test.cc b/pw_bluetooth_profiles/device_info_service_test.cc index 153027e65..5782ef4a7 100644 --- a/pw_bluetooth_profiles/device_info_service_test.cc +++ b/pw_bluetooth_profiles/device_info_service_test.cc @@ -53,8 +53,9 @@ class FakeGattServer final : public bluetooth::gatt::Server { : fake_server_(fake_server) {} // LocalService overrides: - void NotifyValue(const ValueChangedParameters& /* parameters */, - Closure&& /* completion_callback */) override { + void NotifyValue( + const ValueChangedParameters& /* parameters */, + ValueChangedCallback&& /* completion_callback */) override { FAIL(); // Unimplemented } void IndicateValue( diff --git a/pw_build/BUILD.gn b/pw_build/BUILD.gn index 51e0fcc3f..25f53433a 100644 --- a/pw_build/BUILD.gn +++ b/pw_build/BUILD.gn @@ -88,6 +88,18 @@ config("reduced_size") { } } +config("rust_edition_2015") { + rustflags = [ "--edition=2015" ] +} + +config("rust_edition_2018") { + rustflags = [ "--edition=2018" ] +} + +config("rust_edition_2021") { + rustflags = [ "--edition=2021" ] +} + config("strict_warnings") { cflags = [ "-Wall", diff --git a/pw_build/bazel_internal/pigweed_internal.bzl b/pw_build/bazel_internal/pigweed_internal.bzl index ab69f467f..ed452ffe8 100644 --- a/pw_build/bazel_internal/pigweed_internal.bzl +++ b/pw_build/bazel_internal/pigweed_internal.bzl @@ -15,8 +15,8 @@ # the License. """ An internal set of tools for creating embedded CC targets. """ +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain", "use_cpp_toolchain") load("@rules_cc//cc:action_names.bzl", "C_COMPILE_ACTION_NAME") -load("@rules_cc//cc:toolchain_utils.bzl", "find_cpp_toolchain") DEBUGGING = [ "-g", @@ -139,6 +139,6 @@ pw_linker_script = rule( ), "_cc_toolchain": attr.label(default = Label("@bazel_tools//tools/cpp:current_cc_toolchain")), }, - toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], + toolchains = use_cpp_toolchain(), fragments = ["cpp"], ) diff --git a/pw_build/cc_blob_library_test.cc b/pw_build/cc_blob_library_test.cc index 91db07ad1..faa4f6a47 100644 --- a/pw_build/cc_blob_library_test.cc +++ b/pw_build/cc_blob_library_test.cc @@ -21,10 +21,7 @@ namespace pw::build { namespace { static_assert(test::ns::kFirstBlob0123.size() == 4); -static_assert(test::ns::kFirstBlob0123.data() != nullptr); - static_assert(test::ns::kSecondBlob0123.size() == 4); -static_assert(test::ns::kSecondBlob0123.data() != nullptr); TEST(CcBlobLibraryTest, FirstBlobContentsMatch) { EXPECT_EQ(test::ns::kFirstBlob0123[0], std::byte{0}); diff --git a/pw_build/cc_executable.gni b/pw_build/cc_executable.gni index 230b127ee..fe10da498 100644 --- a/pw_build/cc_executable.gni +++ b/pw_build/cc_executable.gni @@ -24,21 +24,6 @@ import("$dir_pw_build/gn_internal/build_target.gni") # templates may need to create pw_source_set targets internally, and can't # import target_types.gni because it creates a circular import path. -declare_args() { - # The name of the GN target type used to build Pigweed executables. - # - # If this is a custom template, the .gni file containing the template must - # be imported at the top of the target configuration file to make it globally - # available. - pw_build_EXECUTABLE_TARGET_TYPE = "executable" - - # The path to the .gni file that defines pw_build_EXECUTABLE_TARGET_TYPE. - # - # If pw_build_EXECUTABLE_TARGET_TYPE is not the default of `executable`, this - # .gni file is imported to provide the template definition. - pw_build_EXECUTABLE_TARGET_TYPE_FILE = "" -} - # This template wraps a configurable target type specified by the current # toolchain to be used for all pw_executable targets. This allows projects to # stamp out unique build logic for each pw_executable target, such as generating diff --git a/pw_build/cc_library.gni b/pw_build/cc_library.gni index 81465c580..02fe625a9 100644 --- a/pw_build/cc_library.gni +++ b/pw_build/cc_library.gni @@ -23,16 +23,6 @@ import("$dir_pw_build/gn_internal/build_target.gni") # templates may need to create pw_source_set targets internally, and can't # import target_types.gni because it creates a circular import path. -declare_args() { - # Additional build targets to add as dependencies for pw_executable, - # pw_static_library, and pw_shared_library targets. The - # $dir_pw_build:link_deps target pulls in these libraries. - # - # pw_build_LINK_DEPS can be used to break circular dependencies for low-level - # libraries such as pw_assert. - pw_build_LINK_DEPS = [] -} - # These templates are wrappers for GN's built-in source_set, static_library, # and shared_library targets. # diff --git a/pw_build/docs.rst b/pw_build/docs.rst index db1964f34..a1df9d929 100644 --- a/pw_build/docs.rst +++ b/pw_build/docs.rst @@ -358,6 +358,9 @@ target. Additionally, it has some of its own arguments: * ``working_directory``: Optional file path. When provided the current working directory will be set to this location before the Python module or script is run. +* ``command_launcher``: Optional string. Arguments to prepend to the Python + command, e.g. ``'/usr/bin/fakeroot --'`` will run the Python script within a + fakeroot environment. * ``venv``: Optional gn target of the pw_python_venv that should be used to run this action. diff --git a/pw_build/generated_pigweed_modules_lists.gni b/pw_build/generated_pigweed_modules_lists.gni index 265324718..77cec9182 100644 --- a/pw_build/generated_pigweed_modules_lists.gni +++ b/pw_build/generated_pigweed_modules_lists.gni @@ -28,6 +28,7 @@ # Declare a build arg for each module. declare_args() { dir_docker = get_path_info("../docker", "abspath") + dir_pw_alignment = get_path_info("../pw_alignment", "abspath") dir_pw_allocator = get_path_info("../pw_allocator", "abspath") dir_pw_analog = get_path_info("../pw_analog", "abspath") dir_pw_android_toolchain = get_path_info("../pw_android_toolchain", "abspath") @@ -152,6 +153,7 @@ declare_args() { dir_pw_thread_freertos = get_path_info("../pw_thread_freertos", "abspath") dir_pw_thread_stl = get_path_info("../pw_thread_stl", "abspath") dir_pw_thread_threadx = get_path_info("../pw_thread_threadx", "abspath") + dir_pw_thread_zephyr = get_path_info("../pw_thread_zephyr", "abspath") dir_pw_tls_client = get_path_info("../pw_tls_client", "abspath") dir_pw_tls_client_boringssl = get_path_info("../pw_tls_client_boringssl", "abspath") @@ -176,6 +178,7 @@ declare_args() { # A list with paths to all Pigweed module. DO NOT SET THIS BUILD ARGUMENT! pw_modules = [ dir_docker, + dir_pw_alignment, dir_pw_allocator, dir_pw_analog, dir_pw_android_toolchain, @@ -292,6 +295,7 @@ declare_args() { dir_pw_thread_freertos, dir_pw_thread_stl, dir_pw_thread_threadx, + dir_pw_thread_zephyr, dir_pw_tls_client, dir_pw_tls_client_boringssl, dir_pw_tls_client_mbedtls, @@ -311,6 +315,7 @@ declare_args() { # A list with all Pigweed module test groups. DO NOT SET THIS BUILD ARGUMENT! pw_module_tests = [ "$dir_docker:tests", + "$dir_pw_alignment:tests", "$dir_pw_allocator:tests", "$dir_pw_analog:tests", "$dir_pw_android_toolchain:tests", @@ -427,6 +432,7 @@ declare_args() { "$dir_pw_thread_freertos:tests", "$dir_pw_thread_stl:tests", "$dir_pw_thread_threadx:tests", + "$dir_pw_thread_zephyr:tests", "$dir_pw_tls_client:tests", "$dir_pw_tls_client_boringssl:tests", "$dir_pw_tls_client_mbedtls:tests", @@ -446,6 +452,7 @@ declare_args() { # A list with all Pigweed modules docs groups. DO NOT SET THIS BUILD ARGUMENT! pw_module_docs = [ "$dir_docker:docs", + "$dir_pw_alignment:docs", "$dir_pw_allocator:docs", "$dir_pw_analog:docs", "$dir_pw_android_toolchain:docs", @@ -562,6 +569,7 @@ declare_args() { "$dir_pw_thread_freertos:docs", "$dir_pw_thread_stl:docs", "$dir_pw_thread_threadx:docs", + "$dir_pw_thread_zephyr:docs", "$dir_pw_tls_client:docs", "$dir_pw_tls_client_boringssl:docs", "$dir_pw_tls_client_mbedtls:docs", diff --git a/pw_build/gn_internal/build_target.gni b/pw_build/gn_internal/build_target.gni index dbdd149ec..149c87eac 100644 --- a/pw_build/gn_internal/build_target.gni +++ b/pw_build/gn_internal/build_target.gni @@ -14,6 +14,29 @@ import("//build_overrides/pigweed.gni") +declare_args() { + # Additional build targets to add as dependencies for pw_executable, + # pw_static_library, and pw_shared_library targets. The + # $dir_pw_build:link_deps target pulls in these libraries. + # + # pw_build_LINK_DEPS can be used to break circular dependencies for low-level + # libraries such as pw_assert. + pw_build_LINK_DEPS = [] + + # The name of the GN target type used to build Pigweed executables. + # + # If this is a custom template, the .gni file containing the template must + # be imported at the top of the target configuration file to make it globally + # available. + pw_build_EXECUTABLE_TARGET_TYPE = "executable" + + # The path to the .gni file that defines pw_build_EXECUTABLE_TARGET_TYPE. + # + # If pw_build_EXECUTABLE_TARGET_TYPE is not the default of `executable`, this + # .gni file is imported to provide the template definition. + pw_build_EXECUTABLE_TARGET_TYPE_FILE = "" +} + # This template is the underlying implementation that defines what makes # pw_source_set, pw_executable, pw_shared_library, and pw_static_library unique. # For more information, see the documentation at @@ -61,6 +84,7 @@ template("pw_internal_build_target") { _builtin_target_types = [ "executable", + "rust_library", "shared_library", "source_set", "static_library", diff --git a/pw_build/platforms/BUILD.bazel b/pw_build/platforms/BUILD.bazel index bd87ad127..554957766 100644 --- a/pw_build/platforms/BUILD.bazel +++ b/pw_build/platforms/BUILD.bazel @@ -71,7 +71,7 @@ platform( platform( name = "cortex_m4_fpu", constraint_values = ["@pigweed_config//:target_rtos"], - parents = ["@bazel_embedded//platforms:cortex_m4"], + parents = ["@bazel_embedded//platforms:cortex_m4_fpu"], ) platform( @@ -117,3 +117,28 @@ platform( constraint_values = ["//pw_build/constraints/board:microbit"], parents = [":nrf52833"], ) + +# --- Test platforms --- + +# This is a platform for compilation testing of freertos backends. This is not +# a complete specification of any real target platform. +platform( + name = "testonly_freertos", + constraint_values = [ + # Use FreeRTOS backends. + "//pw_build/constraints/rtos:freertos", + # Use the FreeRTOS config file for stm32f429i_disc1_stm32cube. + "//targets/stm32f429i_disc1_stm32cube:freertos_config_cv", + # Use the ARM_CM4F port of FreeRTOS. + "@freertos//:port_ARM_CM4F", + # Specify this chipset to use the baremetal pw_sys_io backend (because + # the default pw_sys_io_stdio backend is not compatible with FreeRTOS). + "//pw_build/constraints/chipset:stm32f429", + # os:none means, we're not building for any host platform (Windows, + # Linux, or Mac). The pw_sys_io_baremetal_stm32f429 backend is only + # compatible with os:none. + "@platforms//os:none", + ], + # Inherit from cortex_m4_fpu to use the appropriate Arm toolchain. + parents = [":cortex_m4_fpu"], +) diff --git a/pw_build/py/project_builder_prefs_test.py b/pw_build/py/project_builder_prefs_test.py index 9ede88ff6..f128bee2b 100644 --- a/pw_build/py/project_builder_prefs_test.py +++ b/pw_build/py/project_builder_prefs_test.py @@ -46,7 +46,10 @@ class TestProjectBuilderPrefs(unittest.TestCase): def test_load_no_existing_files(self) -> None: # Create a prefs instance with no loaded config. prefs = ProjectBuilderPrefs( - load_argparse_arguments=add_project_builder_arguments + load_argparse_arguments=add_project_builder_arguments, + project_file=False, + project_user_file=False, + user_file=False, ) # Construct an expected result config. expected_config: Dict[Any, Any] = {} @@ -68,7 +71,10 @@ class TestProjectBuilderPrefs(unittest.TestCase): # Create a prefs instance with the test config file. prefs = ProjectBuilderPrefs( - load_argparse_arguments=add_project_builder_arguments + load_argparse_arguments=add_project_builder_arguments, + project_file=False, + project_user_file=False, + user_file=False, ) # Construct an expected result config. diff --git a/pw_build/py/pw_build/build_recipe.py b/pw_build/py/pw_build/build_recipe.py index 0a777f8f6..ebd4c1b55 100644 --- a/pw_build/py/pw_build/build_recipe.py +++ b/pw_build/py/pw_build/build_recipe.py @@ -315,8 +315,12 @@ class BuildRecipeStatus: _LOG.error(" ╚════════════════════════════════════") _LOG.error('') - def status_slug(self) -> OneStyleAndTextTuple: + def status_slug(self, restarting: bool = False) -> OneStyleAndTextTuple: status = ('', '') + if not self.recipe.enabled: + return ('fg:ansidarkgray', 'Disabled') + + waiting = False if self.done: if self.passed(): status = ('fg:ansigreen', 'OK ') @@ -325,8 +329,12 @@ class BuildRecipeStatus: elif self.started: status = ('fg:ansiyellow', 'Building') else: - status = ('fg:ansigray', 'Waiting ') + waiting = True + status = ('default', 'Waiting ') + # Only show Aborting if the process is building (or has failures). + if restarting and not waiting and not self.passed(): + status = ('fg:ansiyellow', 'Aborting') return status def current_step_formatted(self) -> StyleAndTextTuples: @@ -405,6 +413,7 @@ class BuildRecipe: build_dir: Path steps: List[BuildCommand] = field(default_factory=list) title: Optional[str] = None + enabled: bool = True def __hash__(self): return hash((self.build_dir, self.title, len(self.steps))) @@ -422,6 +431,9 @@ class BuildRecipe: self._status: BuildRecipeStatus = BuildRecipeStatus(self) self.project_builder: Optional['ProjectBuilder'] = None + def toggle_enabled(self) -> None: + self.enabled = not self.enabled + def set_project_builder(self, project_builder) -> None: self.project_builder = project_builder diff --git a/pw_build/py/pw_build/create_python_tree.py b/pw_build/py/pw_build/create_python_tree.py index 9771ff3f6..981f12d5d 100644 --- a/pw_build/py/pw_build/create_python_tree.py +++ b/pw_build/py/pw_build/create_python_tree.py @@ -396,7 +396,6 @@ def _main(): if args.setupcfg_common_file or ( args.setupcfg_override_name and args.setupcfg_override_version ): - config = load_common_config( common_config=args.setupcfg_common_file, package_name_override=args.setupcfg_override_name, diff --git a/pw_build/py/pw_build/generate_modules_lists.py b/pw_build/py/pw_build/generate_modules_lists.py index fbcdbe349..ae9eb9505 100644 --- a/pw_build/py/pw_build/generate_modules_lists.py +++ b/pw_build/py/pw_build/generate_modules_lists.py @@ -23,6 +23,7 @@ Used by modules.gni to generate: import argparse import difflib +import enum import io import os from pathlib import Path @@ -181,6 +182,12 @@ def _missing_modules(root: Path, modules: Sequence[str]) -> Sequence[str]: ) +class Mode(enum.Enum): + WARN = 0 # Warn if anything is out of date + CHECK = 1 # Fail if anything is out of date + UPDATE = 2 # Update the generated modules lists + + def _parse_args() -> dict: parser = argparse.ArgumentParser( description=__doc__, @@ -190,11 +197,13 @@ def _parse_args() -> dict: parser.add_argument('modules_list', type=Path, help='Input modules list') parser.add_argument('modules_gni_file', type=Path, help='Output .gni file') parser.add_argument( - '--warn-only', + '--mode', type=Mode.__getitem__, choices=Mode, required=True + ) + parser.add_argument( + '--stamp', type=Path, - help='Only check PIGWEED_MODULES; takes a path to a stamp file to use', + help='Stamp file for operations that should only run once (warn)', ) - return vars(parser.parse_args()) @@ -202,7 +211,8 @@ def main( root: Path, modules_list: Path, modules_gni_file: Path, - warn_only: Optional[Path], + mode: Mode, + stamp: Optional[Path] = None, ) -> int: """Manages the list of Pigweed modules.""" prefix = Path(os.path.relpath(root, modules_gni_file.parent)) @@ -215,7 +225,7 @@ def main( modules.sort() # Sort in case the modules list in case it wasn't sorted. # Check if the contents of the .gni file are out of date. - if warn_only: + if mode in (Mode.WARN, Mode.CHECK): text = io.StringIO() for line in _generate_modules_gni(prefix, modules): print(line, file=text) @@ -252,7 +262,7 @@ def main( errors.append('\n'.join(diff)) errors.append('\n') - elif not warnings: # Update the modules .gni file. + elif mode is Mode.UPDATE: # Update the modules .gni file with modules_gni_file.open('w', encoding='utf-8') as file: for line in _generate_modules_gni(prefix, modules): print(line, file=file) @@ -270,14 +280,19 @@ def main( # Delete the stamp so this always reruns. Deleting is necessary since # some of the checks do not depend on input files. - if warn_only and warn_only.exists(): - warn_only.unlink() + if stamp and stamp.exists(): + stamp.unlink() + + if mode is Mode.WARN: + return 0 + + if mode is Mode.CHECK: + return 1 - # Warnings are non-fatal if warn_only is True. - return 1 if errors or not warn_only else 0 + return 1 if errors else 0 # Allow warnings but not errors when updating - if warn_only: - warn_only.touch() + if stamp: + stamp.touch() return 0 diff --git a/pw_build/py/pw_build/project_builder.py b/pw_build/py/pw_build/project_builder.py index b8b0e2f60..cb8ab5941 100644 --- a/pw_build/py/pw_build/project_builder.py +++ b/pw_build/py/pw_build/project_builder.py @@ -730,7 +730,10 @@ class ProjectBuilder: # pylint: disable=too-many-instance-attributes return # If restarting or interrupted. if BUILDER_CONTEXT.interrupted(): - _LOG.info(self.color.yellow('Exited due to keyboard interrupt.')) + if BUILDER_CONTEXT.ctrl_c_pressed: + _LOG.info( + self.color.yellow('Exited due to keyboard interrupt.') + ) return # If any build is still pending. if any(recipe.status.pending() for recipe in self): @@ -770,7 +773,7 @@ class ProjectBuilder: # pylint: disable=too-many-instance-attributes logger.info(' ╔════════════════════════════════════') logger.info(' ║') - for (slug, cmd) in zip(build_status, build_descriptions): + for slug, cmd in zip(build_status, build_descriptions): logger.info(' ║ %s %s', slug, cmd) logger.info(' ║') @@ -788,6 +791,11 @@ class ProjectBuilder: # pylint: disable=too-many-instance-attributes def run_recipe( index: int, project_builder: ProjectBuilder, cfg: BuildRecipe, env ) -> bool: + if BUILDER_CONTEXT.interrupted(): + return False + if not cfg.enabled: + return False + num_builds = len(project_builder) index_message = f'[{index}/{num_builds}]' diff --git a/pw_build/py/pw_build/project_builder_context.py b/pw_build/py/pw_build/project_builder_context.py index 1a2adc1e4..f45942351 100644 --- a/pw_build/py/pw_build/project_builder_context.py +++ b/pw_build/py/pw_build/project_builder_context.py @@ -92,7 +92,9 @@ class BuildStatus(Formatter): continue build_status: StyleAndTextTuples = [] - build_status.append(cfg.status.status_slug()) + build_status.append( + cfg.status.status_slug(restarting=self.ctx.restart_flag) + ) build_status.append(('', ' ')) build_status.extend(cfg.status.current_step_formatted()) @@ -203,7 +205,7 @@ class ProjectBuilderContext: # pylint: disable=too-many-instance-attributes,too bottom_toolbar=self.bottom_toolbar, cancel_callback=self.ctrl_c_interrupt, ) - self.progress_bar.__enter__() + self.progress_bar.__enter__() # pylint: disable=unnecessary-dunder-call self.create_title_bar_container() self.progress_bar.app.layout.container.children[ # type: ignore @@ -214,7 +216,7 @@ class ProjectBuilderContext: # pylint: disable=too-many-instance-attributes,too def exit_progress(self) -> None: if not self.progress_bar: return - self.progress_bar.__exit__() + self.progress_bar.__exit__() # pylint: disable=unnecessary-dunder-call def clear_progress_scrollback(self) -> None: if not self.progress_bar: @@ -230,17 +232,15 @@ class ProjectBuilderContext: # pylint: disable=too-many-instance-attributes,too self.progress_bar.invalidate() def get_title_style(self) -> str: + if self.restart_flag: + return 'fg:ansiyellow' + + # Assume passing style = 'fg:ansigreen' if self.current_state == ProjectBuilderState.BUILDING: style = 'fg:ansiyellow' - if ( - self.current_state != ProjectBuilderState.IDLE - and self.interrupted() - ): - return 'fg:ansiyellow' - for cfg in self.recipes: if cfg.status.failed(): style = 'fg:ansired' @@ -267,10 +267,7 @@ class ProjectBuilderContext: # pylint: disable=too-many-instance-attributes,too if cfg.status.done: done_count += 1 - if ( - self.current_state != ProjectBuilderState.IDLE - and self.interrupted() - ): + if self.restart_flag: title = 'INTERRUPT' elif fail_count > 0: title = f'FAILED ({fail_count})' @@ -438,13 +435,21 @@ class ProjectBuilderContext: # pylint: disable=too-many-instance-attributes,too ) -> None: """Exit function called when the user presses ctrl-c.""" + # Note: The correct way to exit Python is via sys.exit() however this + # takes a number of seconds when running pw_watch with multiple parallel + # builds. Instead, this function calls os._exit() to shutdown + # immediately. This is similar to `pw_watch.watch._exit`: + # https://cs.opensource.google/pigweed/pigweed/+/main:pw_watch/py/pw_watch/watch.py?q=_exit.code + if not self.progress_bar: self.restore_logging_and_shutdown(log_after_shutdown) + logging.shutdown() os._exit(exit_code) # pylint: disable=protected-access # Shut everything down after the progress_bar exits. def _really_exit(future: asyncio.Future) -> NoReturn: self.restore_logging_and_shutdown(log_after_shutdown) + logging.shutdown() os._exit(future.result()) # pylint: disable=protected-access if self.progress_bar.app.future: diff --git a/pw_build/py/pw_build/project_builder_prefs.py b/pw_build/py/pw_build/project_builder_prefs.py index a6c3515a7..a13534d9c 100644 --- a/pw_build/py/pw_build/project_builder_prefs.py +++ b/pw_build/py/pw_build/project_builder_prefs.py @@ -16,9 +16,10 @@ import argparse import copy import shlex +from pathlib import Path from typing import Any, Callable, Dict, List, Tuple, Union -from pw_cli.toml_config_loader_mixin import TomlConfigLoaderMixin +from pw_cli.toml_config_loader_mixin import YamlConfigLoaderMixin _DEFAULT_CONFIG: Dict[Any, Any] = { # Config settings not available as a command line options go here. @@ -34,6 +35,10 @@ _DEFAULT_CONFIG: Dict[Any, Any] = { }, } +_DEFAULT_PROJECT_FILE = Path('$PW_PROJECT_ROOT/.pw_build.yaml') +_DEFAULT_PROJECT_USER_FILE = Path('$PW_PROJECT_ROOT/.pw_build.user.yaml') +_DEFAULT_USER_FILE = Path('$HOME/.pw_build.yaml') + def load_defaults_from_argparse( add_parser_arguments: Callable[ @@ -51,7 +56,7 @@ def load_defaults_from_argparse( return defaults_flags -class ProjectBuilderPrefs(TomlConfigLoaderMixin): +class ProjectBuilderPrefs(YamlConfigLoaderMixin): """Pigweed Watch preferences storage class.""" def __init__( @@ -59,15 +64,17 @@ class ProjectBuilderPrefs(TomlConfigLoaderMixin): load_argparse_arguments: Callable[ [argparse.ArgumentParser], argparse.ArgumentParser ], + project_file: Union[Path, bool] = _DEFAULT_PROJECT_FILE, + project_user_file: Union[Path, bool] = _DEFAULT_PROJECT_USER_FILE, + user_file: Union[Path, bool] = _DEFAULT_USER_FILE, ) -> None: self.load_argparse_arguments = load_argparse_arguments self.config_init( config_section_title='pw_build', - # Don't load any config files - project_file=False, - project_user_file=False, - user_file=False, + project_file=project_file, + project_user_file=project_user_file, + user_file=user_file, default_config=_DEFAULT_CONFIG, environment_var='PW_BUILD_CONFIG_FILE', ) diff --git a/pw_build/py/pw_build/python_runner.py b/pw_build/py/pw_build/python_runner.py index efaa77867..2bd8ac5c6 100755 --- a/pw_build/py/pw_build/python_runner.py +++ b/pw_build/py/pw_build/python_runner.py @@ -110,6 +110,9 @@ def _parse_args() -> argparse.Namespace: help='Path to a virtualenv json config to use for this action.', ) parser.add_argument( + '--command-launcher', help='Arguments to prepend to Python command' + ) + parser.add_argument( 'original_cmd', nargs=argparse.REMAINDER, help='Python script with arguments to run', @@ -200,6 +203,7 @@ def main( # pylint: disable=too-many-arguments,too-many-branches,too-many-local capture_output: bool, touch: Optional[Path], working_directory: Optional[Path], + command_launcher: Optional[str], lockfile: Optional[Path], ) -> int: """Script entry point.""" @@ -258,6 +262,9 @@ def main( # pylint: disable=too-many-arguments,too-many-branches,too-many-local if python_interpreter is not None: command = [str(root_build_dir / python_interpreter)] + if command_launcher is not None: + command = shlex.split(command_launcher) + command + if module is not None: command += ['-m', module] diff --git a/pw_build/py/setup.cfg b/pw_build/py/setup.cfg index 56d0edd8e..37b8b733b 100644 --- a/pw_build/py/setup.cfg +++ b/pw_build/py/setup.cfg @@ -22,15 +22,14 @@ description = Python scripts that support the GN build packages = find: zip_safe = False install_requires = - build==0.8.0 + # NOTE: These requirements should stay as >= the lowest version we support. + build>=0.8.0 wheel coverage setuptools types-setuptools - # NOTE: mypy needs to stay in sync with mypy-protobuf - # Currently using mypy 0.991 and mypy-protobuf 3.3.0 (see constraint.list) mypy>=0.971 - pylint==2.9.3 + pylint>=2.9.3 [options.entry_points] console_scripts = diff --git a/pw_build/python_action.gni b/pw_build/python_action.gni index 11028af6c..0c7662774 100644 --- a/pw_build/python_action.gni +++ b/pw_build/python_action.gni @@ -53,6 +53,10 @@ import("$dir_pw_build/python_gn_args.gni") # working_directory Switch to the provided working directory before running # the Python script or action. # +# command_launcher Arguments to prepend to the Python command, e.g. +# '/usr/bin/fakeroot --' to run the Python script within a +# fakeroot environment. +# # venv Optional gn target of the pw_python_venv that should be used # to run this action. # @@ -145,6 +149,13 @@ template("pw_python_action") { ] } + if (defined(invoker.command_launcher)) { + _script_args += [ + "--command-launcher", + invoker.command_launcher, + ] + } + if (defined(invoker._pw_action_type)) { _action_type = invoker._pw_action_type } else { diff --git a/pw_build/rust_executable.gni b/pw_build/rust_executable.gni new file mode 100644 index 000000000..cc564b65d --- /dev/null +++ b/pw_build/rust_executable.gni @@ -0,0 +1,56 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import("//build_overrides/pigweed.gni") + +import("$dir_pw_build/gn_internal/build_target.gni") + +# Note: In general, prefer to import target_types.gni rather than this file. +# +# This template wraps a configurable target type specified by the current +# toolchain to be used for all pw_rust_executable targets. This allows projects +# to stamp out unique build logic for each pw_rust_executable target. +# This wrapper is analogous to pw_executable with additions to support rust +# specific parameters such as rust edition and cargo config features. +# +# Default configs, default visibility, and link deps are applied to the target +# before forwarding to the underlying type as specified by +# pw_build_EXECUTABLE_TARGET_TYPE. +# +# For more information on the features provided by this template, see the full +# docs at https://pigweed.dev/pw_build/?highlight=pw_rust_executable. +template("pw_rust_executable") { + pw_internal_build_target(target_name) { + forward_variables_from(invoker, "*") + + _edition = "2021" + if (defined(invoker.edition)) { + _edition = invoker.edition + } + assert(_edition == "2015" || _edition == "2018" || _edition == "2021", + "edition ${_edition} is not supported") + + if (defined(invoker.configs)) { + configs = invoker.configs + } else { + configs = [] + } + configs += [ "$dir_pw_build:rust_edition_${_edition}" ] + + underlying_target_type = pw_build_EXECUTABLE_TARGET_TYPE + target_type_file = pw_build_EXECUTABLE_TARGET_TYPE_FILE + output_dir = "${target_out_dir}/bin" + add_global_link_deps = true + } +} diff --git a/pw_build/rust_library.gni b/pw_build/rust_library.gni new file mode 100644 index 000000000..370f47d53 --- /dev/null +++ b/pw_build/rust_library.gni @@ -0,0 +1,65 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import("//build_overrides/pigweed.gni") + +import("$dir_pw_build/gn_internal/build_target.gni") + +# Note: In general, prefer to import target_types.gni rather than this file. +# +# This template wraps a configurable target type specified by the current +# toolchain to be used for all pw_rust_library targets. A wrapper for GN's +# built-in rust_library target, it is analogous to pw_static_library with +# additions to support rust specific parameters such as rust edition and cargo +# config features. +# +# For more information on the features provided by this template, see the full +# docs at https://pigweed.dev/pw_build/?highlight=pw_rust_library +template("pw_rust_library") { + pw_internal_build_target(target_name) { + forward_variables_from(invoker, "*") + + crate_name = target_name + if (defined(name)) { + crate_name = name + } + + _edition = "2021" + if (defined(edition)) { + _edition = edition + } + assert(_edition == "2015" || _edition == "2018" || _edition == "2021", + "edition ${_edition} is not supported") + + if (!defined(configs)) { + configs = [] + } + configs += [ "$dir_pw_build:rust_edition_${_edition}" ] + + if (!defined(rustflags)) { + rustflags = [] + } + if (defined(features)) { + foreach(i, features) { + rustflags += [ "--cfg=feature=\"${i}\"" ] + } + } + + underlying_target_type = "rust_library" + crate_name = string_replace(crate_name, "-", "_") + output_name = crate_name + output_dir = "${target_out_dir}/lib" + add_global_link_deps = true + } +} diff --git a/pw_build/target_types.gni b/pw_build/target_types.gni index afb03809d..3803c7194 100644 --- a/pw_build/target_types.gni +++ b/pw_build/target_types.gni @@ -16,3 +16,5 @@ import("//build_overrides/pigweed.gni") import("$dir_pw_build/cc_executable.gni") import("$dir_pw_build/cc_library.gni") +import("$dir_pw_build/rust_executable.gni") +import("$dir_pw_build/rust_library.gni") diff --git a/pw_chrono/public/pw_chrono/system_clock.h b/pw_chrono/public/pw_chrono/system_clock.h index 55801ced9..72035fb56 100644 --- a/pw_chrono/public/pw_chrono/system_clock.h +++ b/pw_chrono/public/pw_chrono/system_clock.h @@ -36,8 +36,6 @@ namespace pw::chrono { namespace backend { -/// @var GetSystemClockTickCount -/// /// The ARM AEBI does not permit the opaque 'time_point' to be passed via /// registers, ergo the underlying fundamental type is forward declared. /// A SystemCLock tick has the units of one SystemClock::period duration. @@ -47,8 +45,6 @@ int64_t GetSystemClockTickCount(); } // namespace backend -/// @struct SystemClock -/// /// The `SystemClock` represents an unsteady, monotonic clock. /// /// The epoch of this clock is unspecified and may not be related to wall time @@ -136,8 +132,6 @@ struct SystemClock { } }; -/// @class VirtualSystemCLock -/// /// An abstract interface representing a SystemClock. /// /// This interface allows decoupling code that uses time from the code that @@ -173,6 +167,8 @@ class VirtualSystemClock { static VirtualSystemClock& RealClock(); virtual ~VirtualSystemClock() = default; + + /// Returns the current time. virtual SystemClock::time_point now() = 0; }; diff --git a/pw_chrono/public/pw_chrono/system_timer.h b/pw_chrono/public/pw_chrono/system_timer.h index cf4c51013..0ec9d5e67 100644 --- a/pw_chrono/public/pw_chrono/system_timer.h +++ b/pw_chrono/public/pw_chrono/system_timer.h @@ -19,14 +19,12 @@ namespace pw::chrono { -/// @class SystemTimer +/// The `SystemTimer` allows an `ExpiryCallback` be executed at a set time in +/// the future. /// -/// The SystemTimer allows an ExpiryCallback be executed at a set time in the -/// future. -/// -/// The base SystemTimer only supports a one-shot style timer with a callback. +/// The base `SystemTimer` only supports a one-shot style timer with a callback. /// A periodic timer can be implemented by rescheduling the timer in the -/// callback through InvokeAt(kDesiredPeriod + expired_deadline). +/// callback through `InvokeAt(kDesiredPeriod + expired_deadline)`. /// /// When implementing a periodic layer on top, the user should be mindful of /// handling missed periodic callbacks. They could opt to invoke the callback @@ -38,12 +36,12 @@ class SystemTimer { public: using native_handle_type = backend::NativeSystemTimerHandle; - /// The ExpiryCallback is either invoked from a high priority thread or an + /// The `ExpiryCallback` is either invoked from a high priority thread or an /// interrupt. /// - /// For a given timer instance, its ExpiryCallback will not preempt itself. + /// For a given timer instance, its `ExpiryCallback` will not preempt itself. /// This makes it appear like there is a single executor of a timer instance's - /// ExpiryCallback. + /// `ExpiryCallback`. /// /// Ergo ExpiryCallbacks should be treated as if they are executed by an /// interrupt, meaning: @@ -56,7 +54,7 @@ class SystemTimer { /// Constructs the SystemTimer based on the user provided /// `pw::Function<void(SystemClock::time_point expired_deadline)>`. Note that - /// The ExpiryCallback is either invoked from a high priority thread or an + /// The `ExpiryCallback` is either invoked from a high priority thread or an /// interrupt. SystemTimer(ExpiryCallback&& callback); diff --git a/pw_chrono_freertos/BUILD.bazel b/pw_chrono_freertos/BUILD.bazel index dc16bf614..60c9eee0c 100644 --- a/pw_chrono_freertos/BUILD.bazel +++ b/pw_chrono_freertos/BUILD.bazel @@ -38,6 +38,7 @@ pw_cc_library( ], deps = [ "//pw_chrono:epoch", + "@freertos", ], ) @@ -52,8 +53,9 @@ pw_cc_library( deps = [ ":system_clock_headers", "//pw_chrono:system_clock_facade", - # TODO(ewout): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. + "//pw_interrupt:context", + "//pw_sync:interrupt_spin_lock", + "@freertos", ], ) diff --git a/pw_chrono_zephyr/CMakeLists.txt b/pw_chrono_zephyr/CMakeLists.txt index 4ed4d33e1..bea4f6de2 100644 --- a/pw_chrono_zephyr/CMakeLists.txt +++ b/pw_chrono_zephyr/CMakeLists.txt @@ -14,22 +14,23 @@ include($ENV{PW_ROOT}/pw_build/pigweed.cmake) +pw_add_library(pw_chrono_zephyr.system_clock INTERFACE + HEADERS + public/pw_chrono_zephyr/system_clock_constants.h + public/pw_chrono_zephyr/system_clock_config.h + public/pw_chrono_zephyr/system_clock_inline.h + public_overrides/pw_chrono_backend/system_clock_config.h + public_overrides/pw_chrono_backend/system_clock_inline.h + PUBLIC_INCLUDES + public + public_overrides + PUBLIC_DEPS + pw_function + pw_chrono.epoch + pw_chrono.system_clock.facade +) + if(CONFIG_PIGWEED_CHRONO_SYSTEM_CLOCK) - pw_add_library(pw_chrono_zephyr.system_clock INTERFACE - HEADERS - public/pw_chrono_zephyr/system_clock_constants.h - public/pw_chrono_zephyr/system_clock_config.h - public/pw_chrono_zephyr/system_clock_inline.h - public_overrides/pw_chrono_backend/system_clock_config.h - public_overrides/pw_chrono_backend/system_clock_inline.h - PUBLIC_INCLUDES - public - public_overrides - PUBLIC_DEPS - pw_function - pw_chrono.epoch - pw_chrono.system_clock.facade - ) zephyr_link_interface(pw_chrono_zephyr.system_clock) zephyr_link_libraries(pw_chrono_zephyr.system_clock) endif() diff --git a/pw_chrono_zephyr/public/pw_chrono_zephyr/system_clock_constants.h b/pw_chrono_zephyr/public/pw_chrono_zephyr/system_clock_constants.h index 4013b5c38..33641d849 100644 --- a/pw_chrono_zephyr/public/pw_chrono_zephyr/system_clock_constants.h +++ b/pw_chrono_zephyr/public/pw_chrono_zephyr/system_clock_constants.h @@ -13,7 +13,7 @@ // the License. #pragma once -#include <kernel.h> +#include <zephyr/kernel.h> #include "pw_chrono/system_clock.h" diff --git a/pw_cli/py/BUILD.bazel b/pw_cli/py/BUILD.bazel index e95742249..c77ce2ca2 100644 --- a/pw_cli/py/BUILD.bazel +++ b/pw_cli/py/BUILD.bazel @@ -51,10 +51,10 @@ py_binary( ) py_test( - name = "plugins_test", + name = "envparse_test", size = "small", srcs = [ - "plugins_test.py", + "envparse_test.py", ], deps = [ ":pw_cli", @@ -62,10 +62,10 @@ py_test( ) py_test( - name = "envparse_test", + name = "plugins_test", size = "small", srcs = [ - "envparse_test.py", + "plugins_test.py", ], deps = [ ":pw_cli", diff --git a/pw_cli/py/BUILD.gn b/pw_cli/py/BUILD.gn index 052ffc6ac..4bb4cd2e3 100644 --- a/pw_cli/py/BUILD.gn +++ b/pw_cli/py/BUILD.gn @@ -46,3 +46,15 @@ pw_python_package("py") { pylintrc = "$dir_pigweed/.pylintrc" mypy_ini = "$dir_pigweed/.mypy.ini" } + +pw_python_script("process_integration_test") { + sources = [ "process_integration_test.py" ] + python_deps = [ ":py" ] + + pylintrc = "$dir_pigweed/.pylintrc" + mypy_ini = "$dir_pigweed/.mypy.ini" + + action = { + stamp = true + } +} diff --git a/pw_cli/py/plugins_test.py b/pw_cli/py/plugins_test.py index 3bafcacf9..a3f35132d 100644 --- a/pw_cli/py/plugins_test.py +++ b/pw_cli/py/plugins_test.py @@ -188,7 +188,6 @@ class TestPluginRegistry(unittest.TestCase): sys.modules[fake_module_name] = fake_module try: - function = lambda: None function.__module__ = fake_module_name self.assertIsNotNone(self._registry.register('a', function)) diff --git a/pw_cli/py/process_integration_test.py b/pw_cli/py/process_integration_test.py new file mode 100644 index 000000000..3d05b1a1b --- /dev/null +++ b/pw_cli/py/process_integration_test.py @@ -0,0 +1,73 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Tests for pw_cli.process.""" + +import unittest +import sys +import textwrap + +from pw_cli import process + +import psutil # type: ignore + + +FAST_TIMEOUT_SECONDS = 0.1 +KILL_SIGNALS = set({-9, 137}) +PYTHON = sys.executable + + +class RunTest(unittest.TestCase): + """Tests for process.run.""" + + def test_returns_output(self) -> None: + echo_str = 'foobar' + print_str = f'print("{echo_str}")' + result = process.run(PYTHON, '-c', print_str) + self.assertEqual(result.output, b'foobar\n') + + def test_timeout_kills_process(self) -> None: + sleep_100_seconds = 'import time; time.sleep(100)' + result = process.run( + PYTHON, '-c', sleep_100_seconds, timeout=FAST_TIMEOUT_SECONDS + ) + self.assertIn(result.returncode, KILL_SIGNALS) + + def test_timeout_kills_subprocess(self) -> None: + # Spawn a subprocess which waits for 100 seconds, print its pid, + # then wait for 100 seconds. + sleep_in_subprocess = textwrap.dedent( + f""" + import subprocess + import time + + child = subprocess.Popen( + ['{PYTHON}', '-c', 'import time; print("booh"); time.sleep(100)'] + ) + print(child.pid, flush=True) + time.sleep(100) + """ + ) + result = process.run( + PYTHON, '-c', sleep_in_subprocess, timeout=FAST_TIMEOUT_SECONDS + ) + self.assertIn(result.returncode, KILL_SIGNALS) + # THe first line of the output is the PID of the child sleep process. + child_pid_str, sep, remainder = result.output.partition(b'\n') + del sep, remainder + child_pid = int(child_pid_str) + self.assertFalse(psutil.pid_exists(child_pid)) + + +if __name__ == '__main__': + unittest.main() diff --git a/pw_cli/py/pw_cli/process.py b/pw_cli/py/pw_cli/process.py index e683db8b4..98852e35f 100644 --- a/pw_cli/py/pw_cli/process.py +++ b/pw_cli/py/pw_cli/process.py @@ -23,6 +23,9 @@ from typing import Dict, IO, Tuple, Union, Optional import pw_cli.color import pw_cli.log +import psutil # type: ignore + + _COLOR = pw_cli.color.colors() _LOG = logging.getLogger(__name__) @@ -32,7 +35,12 @@ PW_SUBPROCESS_ENV = 'PW_SUBPROCESS' class CompletedProcess: - """Information about a process executed in run_async.""" + """Information about a process executed in run_async. + + Attributes: + pid: The process identifier. + returncode: The return code of the process. + """ def __init__( self, @@ -84,6 +92,56 @@ async def _run_and_log(program: str, args: Tuple[str, ...], env: dict): return process, bytes(output) +async def _kill_process_and_children( + process: 'asyncio.subprocess.Process', +) -> None: + """Kills child processes of a process with PID `pid`.""" + # Look up child processes before sending the kill signal to the parent, + # as otherwise the child lookup would fail. + pid = process.pid + try: + process_util = psutil.Process(pid=pid) + kill_list = list(process_util.children(recursive=True)) + except psutil.NoSuchProcess: + # Creating the kill list raced with parent process completion. + # + # Don't bother cleaning up child processes of parent processes that + # exited on their own. + kill_list = [] + + for proc in kill_list: + try: + proc.kill() + except psutil.NoSuchProcess: + pass + + def wait_for_all() -> None: + for proc in kill_list: + try: + proc.wait() + except psutil.NoSuchProcess: + pass + + # Wait for process completion on the executor to avoid blocking the + # event loop. + loop = asyncio.get_event_loop() + wait_for_children = loop.run_in_executor(None, wait_for_all) + + # Send a kill signal to the main process before waiting for the children + # to complete. + try: + process.kill() + await process.wait() + except ProcessLookupError: + _LOG.debug( + 'Process completed before it could be killed. ' + 'This may have been caused by the killing one of its ' + 'child subprocesses.', + ) + + await wait_for_children + + async def run_async( program: str, *args: str, @@ -93,6 +151,20 @@ async def run_async( ) -> CompletedProcess: """Runs a command, capturing and optionally logging its output. + Args: + program: The program to run in a new process. + args: The arguments to pass to the program. + env: An optional mapping of environment variables within which to run + the process. + log_output: Whether to log stdout and stderr of the process to this + process's stdout (prefixed with the PID of the subprocess from which + the output originated). If unspecified, the child process's stdout + and stderr will be captured, and both will be stored in the returned + `CompletedProcess`'s output`. + timeout: An optional floating point number of seconds to allow the + subprocess to run before killing it and its children. If unspecified, + the subprocess will be allowed to continue exiting until it completes. + Returns a CompletedProcess with details from the process. """ @@ -123,13 +195,12 @@ async def run_async( await asyncio.wait_for(process.wait(), timeout) except asyncio.TimeoutError: _LOG.error('%s timed out after %d seconds', program, timeout) - process.kill() - await process.wait() + await _kill_process_and_children(process) if process.returncode: _LOG.error('%s exited with status %d', program, process.returncode) else: - _LOG.debug('%s exited successfully', program) + _LOG.error('%s exited successfully', program) return CompletedProcess(process, output) diff --git a/pw_cli/py/setup.cfg b/pw_cli/py/setup.cfg index be8343850..20af8de6f 100644 --- a/pw_cli/py/setup.cfg +++ b/pw_cli/py/setup.cfg @@ -21,6 +21,10 @@ description = Pigweed swiss-army knife [options] packages = find: zip_safe = False +install_requires = + psutil + pyyaml + toml [options.entry_points] console_scripts = pw = pw_cli.__main__:main diff --git a/pw_console/py/pw_console/log_filter.py b/pw_console/py/pw_console/log_filter.py index a2c4def57..60411efc1 100644 --- a/pw_console/py/pw_console/log_filter.py +++ b/pw_console/py/pw_console/log_filter.py @@ -92,7 +92,7 @@ class LogFilter: field: Optional[str] = None def pattern(self): - return self.regex.pattern + return self.regex.pattern # pylint: disable=no-member def matches(self, log: LogLine): field = log.ansi_stripped_log @@ -110,7 +110,7 @@ class LogFilter: elif self.field == 'time': field = log.record.asctime - match = self.regex.search(field) + match = self.regex.search(field) # pylint: disable=no-member if self.invert: return not match @@ -143,7 +143,9 @@ class LogFilter: apply_highlighting(exploded_fragments, i) else: # Highlight each non-overlapping search match. - for match in self.regex.finditer(line_text): + for match in self.regex.finditer( # pylint: disable=no-member + line_text + ): # pylint: disable=no-member for fragment_i in range(match.start(), match.end()): apply_highlighting(exploded_fragments, fragment_i) diff --git a/pw_console/py/pw_console/log_line.py b/pw_console/py/pw_console/log_line.py index 6543e30ae..0277166af 100644 --- a/pw_console/py/pw_console/log_line.py +++ b/pw_console/py/pw_console/log_line.py @@ -43,7 +43,9 @@ class LogLine: """Parse log metadata fields from various sources.""" # 1. Parse any metadata from the message itself. - self.metadata = FormatStringWithMetadata(str(self.record.message)) + self.metadata = FormatStringWithMetadata( + str(self.record.message) # pylint: disable=no-member + ) # pylint: disable=no-member self.formatted_log = self.formatted_log.replace( self.metadata.raw_string, self.metadata.message ) @@ -65,9 +67,9 @@ class LogLine: # See: # https://docs.python.org/3/library/logging.html#logging.debug if hasattr(self.record, 'extra_metadata_fields') and ( - self.record.extra_metadata_fields # type: ignore + self.record.extra_metadata_fields # type: ignore # pylint: disable=no-member ): - fields = self.record.extra_metadata_fields # type: ignore + fields = self.record.extra_metadata_fields # type: ignore # pylint: disable=no-member for key, value in fields.items(): self.metadata.fields[key] = value diff --git a/pw_console/py/pw_console/log_pane.py b/pw_console/py/pw_console/log_pane.py index 36c0463c3..c9bfa8f84 100644 --- a/pw_console/py/pw_console/log_pane.py +++ b/pw_console/py/pw_console/log_pane.py @@ -290,7 +290,6 @@ class LogContentControl(UIControl): mouse_event.event_type == MouseEventType.MOUSE_UP and mouse_event.button == MouseButton.LEFT ): - # If a drag was in progress and this is the first mouse release # press, set the stop flag. if ( diff --git a/pw_console/py/pw_console/log_screen.py b/pw_console/py/pw_console/log_screen.py index ebf530033..2e0d5f26d 100644 --- a/pw_console/py/pw_console/log_screen.py +++ b/pw_console/py/pw_console/log_screen.py @@ -228,7 +228,6 @@ class LogScreen: # Loop through a copy of the line_buffer in case it is mutated before # this function is complete. for i, line in enumerate(list(self.line_buffer)): - # Is this line the cursor_position? Apply line highlighting if ( i == self.cursor_position diff --git a/pw_console/py/pw_console/pigweed_code_style.py b/pw_console/py/pw_console/pigweed_code_style.py index 689ee6771..a21cd6829 100644 --- a/pw_console/py/pw_console/pigweed_code_style.py +++ b/pw_console/py/pw_console/pigweed_code_style.py @@ -119,7 +119,6 @@ def swap_light_dark(color: str) -> str: class PigweedCodeStyle(Style): - background_color = '#2e2e2e' default_style = '' @@ -127,7 +126,6 @@ class PigweedCodeStyle(Style): class PigweedCodeLightStyle(Style): - background_color = '#f8f8f8' default_style = '' diff --git a/pw_console/py/pw_console/plugins/twenty48_pane.py b/pw_console/py/pw_console/plugins/twenty48_pane.py index 104e92181..891b2481d 100644 --- a/pw_console/py/pw_console/plugins/twenty48_pane.py +++ b/pw_console/py/pw_console/plugins/twenty48_pane.py @@ -374,7 +374,6 @@ class Twenty48Pane(FloatingWindowPane, PluginMixin): """ def __init__(self, include_resize_handle: bool = True, **kwargs): - super().__init__( pane_title='2048', height=Dimension(preferred=17), diff --git a/pw_console/py/pw_console/progress_bar/progress_bar_impl.py b/pw_console/py/pw_console/progress_bar/progress_bar_impl.py index d68d8255c..08dd62d55 100644 --- a/pw_console/py/pw_console/progress_bar/progress_bar_impl.py +++ b/pw_console/py/pw_console/progress_bar/progress_bar_impl.py @@ -59,7 +59,6 @@ class TextIfNotHidden(Text): ) -> AnyFormattedText: formatted_text = super().format(progress_bar, progress, width) if hasattr(progress, 'hide_eta') and progress.hide_eta: # type: ignore - formatted_text = [('', ' ' * width)] return formatted_text @@ -99,7 +98,6 @@ class ProgressBarImpl: formatters: Optional[Sequence[Formatter]] = None, style: Optional[BaseStyle] = None, ) -> None: - self.title = title self.formatters = formatters or create_default_formatters() self.counters: List[ProgressBarCounter[object]] = [] diff --git a/pw_console/py/pw_console/progress_bar/progress_bar_state.py b/pw_console/py/pw_console/progress_bar/progress_bar_state.py index 37bcc068e..159cc3165 100644 --- a/pw_console/py/pw_console/progress_bar/progress_bar_state.py +++ b/pw_console/py/pw_console/progress_bar/progress_bar_state.py @@ -76,7 +76,7 @@ class ProgressBarState: # Shut down the ProgressBar prompt_toolkit application prog_bar = self.instance if prog_bar is not None and hasattr(prog_bar, '__exit__'): - prog_bar.__exit__() + prog_bar.__exit__() # pylint: disable=unnecessary-dunder-call raise KeyboardInterrupt signal.signal(signal.SIGINT, handle_sigint) @@ -95,7 +95,7 @@ class ProgressBarState: ) # Start the ProgressBar prompt_toolkit application in a separate # thread. - prog_bar.__enter__() + prog_bar.__enter__() # pylint: disable=unnecessary-dunder-call self.instance = prog_bar return self.instance diff --git a/pw_console/py/pw_console/pw_ptpython_repl.py b/pw_console/py/pw_console/pw_ptpython_repl.py index c9b2f4a54..03f405424 100644 --- a/pw_console/py/pw_console/pw_ptpython_repl.py +++ b/pw_console/py/pw_console/pw_ptpython_repl.py @@ -95,7 +95,6 @@ class PwPtPythonRepl( extra_completers: Optional[Iterable] = None, **ptpython_kwargs, ): - completer = None if extra_completers: # Create the default python completer used by @@ -279,7 +278,7 @@ class PwPtPythonRepl( # Trigger a prompt_toolkit application redraw. self.repl_pane.application.application.invalidate() - async def _run_system_command( + async def _run_system_command( # pylint: disable=no-self-use self, text, stdout_proxy, _stdin_proxy ) -> int: """Run a shell command and print results to the repl.""" diff --git a/pw_console/py/pw_console/pyserial_wrapper.py b/pw_console/py/pw_console/pyserial_wrapper.py index fd7467938..57c297a52 100644 --- a/pw_console/py/pw_console/pyserial_wrapper.py +++ b/pw_console/py/pw_console/pyserial_wrapper.py @@ -12,15 +12,20 @@ # License for the specific language governing permissions and limitations under # the License. """Wrapers for pyserial classes to log read and write data.""" +from __future__ import annotations from contextvars import ContextVar import logging import textwrap +from typing import TYPE_CHECKING -import serial # type: ignore +import serial from pw_console.widgets.event_count_history import EventCountHistory +if TYPE_CHECKING: + from _typeshed import ReadableBuffer + _LOG = logging.getLogger('pw_console.serial_debug_logger') @@ -87,8 +92,8 @@ class SerialWithLogging(serial.Serial): # pylint: disable=too-many-ancestors super().__init__(*args, **kwargs) self.pw_bps_history = BANDWIDTH_HISTORY_CONTEXTVAR.get() - def read(self, *args, **kwargs): - data = super().read(*args, **kwargs) + def read(self, size: int = 1) -> bytes: + data = super().read(size) self.pw_bps_history['read'].log(len(data)) self.pw_bps_history['total'].log(len(data)) @@ -126,11 +131,11 @@ class SerialWithLogging(serial.Serial): # pylint: disable=too-many-ancestors return data - def write(self, data: bytes, *args, **kwargs): - self.pw_bps_history['write'].log(len(data)) - self.pw_bps_history['total'].log(len(data)) + def write(self, data: ReadableBuffer) -> None: + if isinstance(data, bytes) and len(data) > 0: + self.pw_bps_history['write'].log(len(data)) + self.pw_bps_history['total'].log(len(data)) - if len(data) > 0: prefix = 'Write %2d B: ' % len(data) _LOG.debug( '%s%s', @@ -162,4 +167,4 @@ class SerialWithLogging(serial.Serial): # pylint: disable=too-many-ancestors ), ) - super().write(data, *args, **kwargs) + super().write(data) diff --git a/pw_console/py/pw_console/repl_pane.py b/pw_console/py/pw_console/repl_pane.py index 9547215c9..20a3b9504 100644 --- a/pw_console/py/pw_console/repl_pane.py +++ b/pw_console/py/pw_console/repl_pane.py @@ -21,6 +21,7 @@ import pprint from dataclasses import dataclass from typing import ( Any, + Awaitable, Callable, Dict, List, @@ -82,8 +83,9 @@ class UserCodeExecution: output: str stdout: str stderr: str - stdout_check_task: Optional[concurrent.futures.Future] = None + stdout_check_task: Optional[Awaitable] = None result_object: Optional[Any] = None + result_str: Optional[str] = None exception_text: Optional[str] = None @property @@ -112,7 +114,7 @@ class ReplPane(WindowPane): ) -> None: super().__init__(application, pane_title) - self.executed_code: List = [] + self.executed_code: List[UserCodeExecution] = [] self.application = application self.pw_ptpython_repl = python_repl @@ -481,7 +483,6 @@ class ReplPane(WindowPane): exception_text='', result_object=None, ): - code = self._get_executed_code(future) if code: code.output = result_text @@ -489,10 +490,14 @@ class ReplPane(WindowPane): code.stderr = stderr_text code.exception_text = exception_text code.result_object = result_object + if result_object is not None: + code.result_str = self._format_result_object(result_object) + self._log_executed_code(code, prefix='FINISH') self.update_output_buffer('repl_pane.append_result_to_executed_code') - def get_output_buffer_text(self, code_items=None, show_index=True): + def _format_result_object(self, result_object: Any) -> str: + """Pretty print format a Python object respecting the window width.""" content_width = ( self.current_pane_width if self.current_pane_width else 80 ) @@ -500,12 +505,19 @@ class ReplPane(WindowPane): indent=2, width=content_width ).pformat + return pprint_respecting_width(result_object) + + def get_output_buffer_text( + self, + code_items: Optional[List[UserCodeExecution]] = None, + show_index: bool = True, + ): executed_code = code_items or self.executed_code template = self.application.get_template('repl_output.jinja') + return template.render( code_items=executed_code, - result_format=pprint_respecting_width, show_index=show_index, ) diff --git a/pw_console/py/pw_console/templates/repl_output.jinja b/pw_console/py/pw_console/templates/repl_output.jinja index 4c482c109..976cb8b1d 100644 --- a/pw_console/py/pw_console/templates/repl_output.jinja +++ b/pw_console/py/pw_console/templates/repl_output.jinja @@ -30,8 +30,8 @@ Running... {% endif %} {% if code.exception_text %} {{ code.exception_text }} -{% elif code.result_object %} -{{ result_format(code.result_object) }} +{% elif code.result_str %} +{{ code.result_str }} {% elif code.output %} {{ code.output }} {% endif %} diff --git a/pw_console/py/pw_console/widgets/window_pane_toolbar.py b/pw_console/py/pw_console/widgets/window_pane_toolbar.py index 354d3d5c1..18ec4cd9a 100644 --- a/pw_console/py/pw_console/widgets/window_pane_toolbar.py +++ b/pw_console/py/pw_console/widgets/window_pane_toolbar.py @@ -184,7 +184,6 @@ class WindowPaneToolbar: include_resize_handle: bool = True, click_to_focus_text: str = 'click to focus', ): - self.parent_window_pane = parent_window_pane self.title = title self.subtitle = subtitle diff --git a/pw_console/py/pw_console/window_manager.py b/pw_console/py/pw_console/window_manager.py index 5a1d24175..817e1d7ff 100644 --- a/pw_console/py/pw_console/window_manager.py +++ b/pw_console/py/pw_console/window_manager.py @@ -953,7 +953,6 @@ class WindowManager: for logger_name, logger_options in window_options.get( 'loggers', {} ).items(): - log_level_name = logger_options.get('level', None) new_pane.add_log_handler(logger_name, level_name=log_level_name) return new_pane diff --git a/pw_console/py/setup.cfg b/pw_console/py/setup.cfg index 97b921a15..60021a431 100644 --- a/pw_console/py/setup.cfg +++ b/pw_console/py/setup.cfg @@ -28,9 +28,10 @@ install_requires = ptpython>=3.0.20 pygments pyperclip - pyserial + pyserial>=3.5,<4.0 pyyaml types-pygments + types-pyserial>=3.5,<4.0 types-pyyaml websockets diff --git a/pw_console/testing.rst b/pw_console/testing.rst index 38fb9f4b4..af91318fb 100644 --- a/pw_console/testing.rst +++ b/pw_console/testing.rst @@ -731,23 +731,36 @@ Python Repl & Output * - 5 - | Enter the following text and press :kbd:`Enter` to run - | ``globals()`` + | ``locals()`` - | The results should appear pretty printed - |checkbox| * - 6 + - | Enter the following text and press :kbd:`Enter` to run + | ``zzzz = 'test'`` + - | No new results are shown + | The previous ``locals()`` output does not show ``'zzzz': 'test'`` + - |checkbox| + + * - 7 + - | Enter the following text and press :kbd:`Enter` to run + | ``locals()`` + - | The output ends with ``'zzzz': 'test'}`` + - |checkbox| + + * - 8 - | With the cursor over the Python Results, | use the mouse wheel to scroll up and down. - | The output window should be able to scroll all | the way to the beginning and end of the buffer. - |checkbox| - * - 7 + * - 9 - Click empty whitespace in the ``Python Repl`` window - Python Repl pane is focused - |checkbox| - * - 8 + * - 10 - | Enter the following text and press :kbd:`Enter` to run | ``!ls`` - | 1. Shell output of running the ``ls`` command should appear in the diff --git a/pw_crypto/BUILD.gn b/pw_crypto/BUILD.gn index 9787100ee..919ab7bf0 100644 --- a/pw_crypto/BUILD.gn +++ b/pw_crypto/BUILD.gn @@ -19,6 +19,7 @@ import("$dir_pw_build/facade.gni") import("$dir_pw_build/target_types.gni") import("$dir_pw_crypto/backend.gni") import("$dir_pw_docgen/docs.gni") +import("$dir_pw_third_party/micro_ecc/micro_ecc.gni") import("$dir_pw_unit_test/test.gni") config("default_config") { @@ -83,6 +84,7 @@ pw_test_group("tests") { ":sha256_test", ":sha256_mock_test", ":ecdsa_test", + ":ecdsa_uecc_little_endian_test", ] } @@ -193,8 +195,28 @@ pw_source_set("ecdsa_uecc") { public_deps = [ ":ecdsa.facade" ] } +pw_source_set("ecdsa_uecc_little_endian") { + sources = [ "ecdsa_uecc.cc" ] + deps = [ + "$dir_pw_log", + "$dir_pw_third_party/micro_ecc:micro_ecc_little_endian", + ] + public_deps = [ ":ecdsa.facade" ] +} + +# This test targets the specific backend pointed to by +# pw_crypto_ECDSA_BACKEND. pw_test("ecdsa_test") { enable_if = pw_crypto_ECDSA_BACKEND != "" deps = [ ":ecdsa" ] sources = [ "ecdsa_test.cc" ] } + +# This test targets the micro_ecc little endian backend specifically. +# +# TODO(b/273819841) deduplicate all backend tests. +pw_test("ecdsa_uecc_little_endian_test") { + enable_if = dir_pw_third_party_micro_ecc != "" + sources = [ "ecdsa_test.cc" ] + deps = [ ":ecdsa_uecc_little_endian" ] +} diff --git a/pw_crypto/docs.rst b/pw_crypto/docs.rst index 601952869..1230cd35d 100644 --- a/pw_crypto/docs.rst +++ b/pw_crypto/docs.rst @@ -99,7 +99,9 @@ configured. # Install and configure MbedTLS pw package install mbedtls gn gen out --args=' - dir_pw_third_party_mbedtls=pw_env_setup_PACKAGE_ROOT + "/mbedtls" + import("//build_overrides/pigweed_environment.gni") + + dir_pw_third_party_mbedtls=pw_env_setup_PACKAGE_ROOT+"/mbedtls" pw_crypto_SHA256_BACKEND="//pw_crypto:sha256_mbedtls" pw_crypto_ECDSA_BACKEND="//pw_crypto:ecdsa_mbedtls" ' @@ -145,7 +147,9 @@ configured. # Install and configure BoringSSL pw package install boringssl gn gen out --args=' - dir_pw_third_party_boringssl=pw_env_setup_PACKAGE_ROOT + "/boringssl" + import("//build_overrides/pigweed_environment.gni") + + dir_pw_third_party_boringssl=pw_env_setup_PACKAGE_ROOT+"/boringssl" pw_crypto_SHA256_BACKEND="//pw_crypto:sha256_boringssl" pw_crypto_ECDSA_BACKEND="//pw_crypto:ecdsa_boringssl" ' @@ -165,10 +169,18 @@ To select Micro ECC, the library needs to be installed and configured. # Install and configure Micro ECC pw package install micro-ecc gn gen out --args=' - dir_pw_third_party_micro_ecc=pw_env_setup_PACKAGE_ROOT + "//micro-ecc" + import("//build_overrides/pigweed_environment.gni") + + dir_pw_third_party_micro_ecc=pw_env_setup_PACKAGE_ROOT+"/micro-ecc" pw_crypto_ECDSA_BACKEND="//pw_crypto:ecdsa_uecc" ' +The default micro-ecc backend uses big endian as is standard practice. It also +has a little-endian configuration which can be used to slightly reduce call +stack frame use and/or when non pw_crypto clients use the same micro-ecc +with a little-endian configuration. The little-endian version of micro-ecc +can be selected with ``pw_crypto_ECDSA_BACKEND="//pw_crypto:ecdsa_uecc_little_endian"`` + Note Micro-ECC does not implement any hashing functions, so you will need to use other backends for SHA256 functionality if needed. Size Reports diff --git a/pw_crypto/ecdsa_uecc.cc b/pw_crypto/ecdsa_uecc.cc index 937d79d56..912a28b44 100644 --- a/pw_crypto/ecdsa_uecc.cc +++ b/pw_crypto/ecdsa_uecc.cc @@ -14,6 +14,8 @@ #define PW_LOG_MODULE_NAME "ECDSA-UECC" #define PW_LOG_LEVEL PW_LOG_LEVEL_WARN +#include <cstring> + #include "pw_crypto/ecdsa.h" #include "pw_log/log.h" #include "uECC.h" @@ -21,33 +23,62 @@ namespace pw::crypto::ecdsa { constexpr size_t kP256CurveOrderBytes = 32; +constexpr size_t kP256PublicKeySize = 2 * kP256CurveOrderBytes + 1; +constexpr size_t kP256SignatureSize = kP256CurveOrderBytes * 2; Status VerifyP256Signature(ConstByteSpan public_key, ConstByteSpan digest, ConstByteSpan signature) { - const uint8_t* public_key_bytes = - reinterpret_cast<const uint8_t*>(public_key.data()); - const uint8_t* digest_bytes = reinterpret_cast<const uint8_t*>(digest.data()); - const uint8_t* signature_bytes = - reinterpret_cast<const uint8_t*>(signature.data()); - - uECC_Curve curve = uECC_secp256r1(); + // Signature expected in raw format (r||s) + if (signature.size() != kP256SignatureSize) { + PW_LOG_DEBUG("Bad signature format"); + return Status::InvalidArgument(); + } // Supports SEC 1 uncompressed form (04||X||Y) only. - if (public_key.size() != (2 * kP256CurveOrderBytes + 1) || - public_key_bytes[0] != 0x04) { + if (public_key.size() != kP256PublicKeySize || + std::to_integer<uint8_t>(public_key.data()[0]) != 0x04) { PW_LOG_DEBUG("Bad public key format"); return Status::InvalidArgument(); } - // Make sure the public key is on the curve. - if (!uECC_valid_public_key(public_key_bytes + 1, curve)) { - return Status::InvalidArgument(); - } +#if defined(uECC_VLI_NATIVE_LITTLE_ENDIAN) && uECC_VLI_NATIVE_LITTLE_ENDIAN + // uECC_VLI_NATIVE_LITTLE_ENDIAN is defined with a non-zero value when + // pw_crypto_ECDSA_BACKEND is set to "//pw_crypto:ecdsa_uecc_little_endian". + // + // Since pw_crypto APIs are big endian only (standard practice), here we + // need to convert input parameters to little endian. + // + // Additionally uECC requires these little endian buffers to be word aligned + // in case unaligned accesses are not supported by the hardware. We choose + // the maximum 8-byte alignment to avoid referrencing internal uECC headers. + alignas(8) uint8_t signature_bytes[kP256SignatureSize]; + memcpy(signature_bytes, signature.data(), sizeof(signature_bytes)); + std::reverse(signature_bytes, signature_bytes + kP256CurveOrderBytes); // r + std::reverse(signature_bytes + kP256CurveOrderBytes, + signature_bytes + sizeof(signature_bytes)); // s - // Signature expected in raw format (r||s) - if (signature.size() != kP256CurveOrderBytes * 2) { - PW_LOG_DEBUG("Bad signature format"); + alignas(8) uint8_t public_key_bytes[kP256PublicKeySize - 1]; + memcpy(public_key_bytes, public_key.data() + 1, sizeof(public_key_bytes)); + std::reverse(public_key_bytes, public_key_bytes + kP256CurveOrderBytes); // X + std::reverse(public_key_bytes + kP256CurveOrderBytes, + public_key_bytes + sizeof(public_key_bytes)); // Y + + alignas(8) uint8_t digest_bytes[kP256CurveOrderBytes]; + memcpy(digest_bytes, digest.data(), sizeof(digest_bytes)); + std::reverse(digest_bytes, digest_bytes + sizeof(digest_bytes)); +#else + const uint8_t* public_key_bytes = + reinterpret_cast<const uint8_t*>(public_key.data()) + 1; + const uint8_t* digest_bytes = reinterpret_cast<const uint8_t*>(digest.data()); + const uint8_t* signature_bytes = + reinterpret_cast<const uint8_t*>(signature.data()); +#endif // uECC_VLI_NATIVE_LITTLE_ENDIAN + + uECC_Curve curve = uECC_secp256r1(); + // Make sure the public key is on the curve. + if (!uECC_valid_public_key(public_key_bytes, curve)) { + PW_LOG_DEBUG("Bad public key curve"); return Status::InvalidArgument(); } @@ -59,7 +90,7 @@ Status VerifyP256Signature(ConstByteSpan public_key, } // Verify the signature. - if (!uECC_verify(public_key_bytes + 1, + if (!uECC_verify(public_key_bytes, digest_bytes, digest.size(), signature_bytes, diff --git a/pw_docgen/docs.rst b/pw_docgen/docs.rst index 5528ea183..6971384a3 100644 --- a/pw_docgen/docs.rst +++ b/pw_docgen/docs.rst @@ -168,6 +168,52 @@ Sphinx Extensions This module houses Pigweed-specific extensions for the Sphinx documentation generator. Extensions are included and configured in ``docs/conf.py``. +module_metadata +--------------- +Per :ref:`SEED-0102 <seed-0102>`, Pigweed module documentation has a standard +format. The ``pigweed-module`` Sphinx directive provides that format and +registers module metadata that can be used elsewhere in the Sphinx build. + +We need to add the directive after the document title, and add a class *to* +the document title to achieve the title & subtitle formatting. Here's an +example: + +.. code-block:: rst + + .. rst-class:: with-subtitle + + ========= + pw_string + ========= + + .. pigweed-module:: + :name: pw_string + :tagline: Efficient, easy, and safe string manipulation + :status: stable + :languages: C++14, C++17 + :code-size-impact: 500 to 1500 bytes + :get-started: module-pw_string-get-started + :design: module-pw_string-design + :guides: module-pw_string-guide + :api: module-pw_string-api + + Module sales pitch goes here! + +Directive options +_________________ +- ``name``: The module name (required) +- ``tagline``: A very short tagline that summarizes the module (required) +- ``status``: One of ``experimental``, ``unstable``, and ``stable`` (required) +- ``is-deprecated``: A flag indicating that the module is deprecated +- ``languages``: A comma-separated list of languages the module supports +- ``code-size-impact``: A summarize of the average code size impact +- ``get-started``: A reference to the getting started section (required) +- ``tutorials``: A reference to the tutorials section +- ``guides``: A reference to the guides section +- ``design``: A reference to the design considerations section (required) +- ``concepts``: A reference to the concepts documentation +- ``api``: A reference to the API documentation + google_analytics ---------------- When this extension is included and a ``google_analytics_id`` is set in the diff --git a/pw_docgen/py/BUILD.gn b/pw_docgen/py/BUILD.gn index 3e6e34fb2..55c48aaeb 100644 --- a/pw_docgen/py/BUILD.gn +++ b/pw_docgen/py/BUILD.gn @@ -27,6 +27,7 @@ pw_python_package("py") { "pw_docgen/docgen.py", "pw_docgen/sphinx/__init__.py", "pw_docgen/sphinx/google_analytics.py", + "pw_docgen/sphinx/module_metadata.py", ] pylintrc = "$dir_pigweed/.pylintrc" mypy_ini = "$dir_pigweed/.mypy.ini" diff --git a/pw_docgen/py/pw_docgen/sphinx/module_metadata.py b/pw_docgen/py/pw_docgen/sphinx/module_metadata.py new file mode 100644 index 000000000..d6fe05497 --- /dev/null +++ b/pw_docgen/py/pw_docgen/sphinx/module_metadata.py @@ -0,0 +1,226 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Sphinx directives for Pigweed module metadata""" + +from typing import List + +import docutils + +# pylint: disable=consider-using-from-import +import docutils.parsers.rst.directives as directives # type: ignore + +# pylint: enable=consider-using-from-import +from sphinx.application import Sphinx as SphinxApplication +from sphinx.util.docutils import SphinxDirective +from sphinx_design.badges_buttons import ButtonRefDirective # type: ignore +from sphinx_design.cards import CardDirective # type: ignore + + +def status_choice(arg): + return directives.choice(arg, ('experimental', 'unstable', 'stable')) + + +def status_badge(module_status: str) -> str: + role = ':bdg-primary:' + return role + f'`{module_status.title()}`' + + +def cs_url(module_name: str): + return f'https://cs.opensource.google/pigweed/pigweed/+/main:{module_name}/' + + +def concat_tags(*tag_lists: List[str]): + all_tags = tag_lists[0] + + for tag_list in tag_lists[1:]: + if len(tag_list) > 0: + all_tags.append(':octicon:`dot-fill`') + all_tags.extend(tag_list) + + return all_tags + + +class PigweedModuleDirective(SphinxDirective): + """Directive registering module metadata, rendering title & info card.""" + + required_arguments = 0 + final_argument_whitespace = True + has_content = True + option_spec = { + 'name': directives.unchanged_required, + 'tagline': directives.unchanged_required, + 'status': status_choice, + 'is-deprecated': directives.flag, + 'languages': directives.unchanged, + 'code-size-impact': directives.unchanged, + 'facade': directives.unchanged, + 'get-started': directives.unchanged_required, + 'tutorials': directives.unchanged, + 'guides': directives.unchanged, + 'concepts': directives.unchanged, + 'design': directives.unchanged_required, + 'api': directives.unchanged, + } + + def try_get_option(self, option: str): + try: + return self.options[option] + except KeyError: + raise self.error(f' :{option}: option is required') + + def maybe_get_option(self, option: str): + try: + return self.options[option] + except KeyError: + return None + + def create_section_button(self, title: str, ref: str): + node = docutils.nodes.list_item(classes=['pw-module-section-button']) + node += ButtonRefDirective( + name='', + arguments=[ref], + options={'color': 'primary'}, + content=[title], + lineno=0, + content_offset=0, + block_text='', + state=self.state, + state_machine=self.state_machine, + ).run() + + return node + + def register_metadata(self): + module_name = self.try_get_option('name') + + if 'facade' in self.options: + facade = self.options['facade'] + + # Initialize the module relationship dict if needed + if not hasattr(self.env, 'pw_module_relationships'): + self.env.pw_module_relationships = {} + + # Initialize the backend list for this facade if needed + if facade not in self.env.pw_module_relationships: + self.env.pw_module_relationships[facade] = [] + + # Add this module as a backend of the provided facade + self.env.pw_module_relationships[facade].append(module_name) + + if 'is-deprecated' in self.options: + # Initialize the deprecated modules list if needed + if not hasattr(self.env, 'pw_modules_deprecated'): + self.env.pw_modules_deprecated = [] + + self.env.pw_modules_deprecated.append(module_name) + + def run(self): + tagline = docutils.nodes.paragraph( + text=self.try_get_option('tagline'), + classes=['section-subtitle'], + ) + + status_tags: List[str] = [ + status_badge(self.try_get_option('status')), + ] + + if 'is-deprecated' in self.options: + status_tags.append(':bdg-danger:`Deprecated`') + + language_tags = [] + + if 'languages' in self.options: + languages = self.options['languages'].split(',') + + if len(languages) > 0: + for language in languages: + language_tags.append(f':bdg-info:`{language.strip()}`') + + code_size_impact = [] + + if code_size_text := self.maybe_get_option('code-size-impact'): + code_size_impact.append(f'**Code Size Impact:** {code_size_text}') + + # Move the directive content into a section that we can render wherever + # we want. + content = docutils.nodes.paragraph() + self.state.nested_parse(self.content, 0, content) + + # The card inherits its content from this node's content, which we've + # already pulled out. So we can replace this node's content with the + # content we need in the card. + self.content = docutils.statemachine.StringList( + concat_tags(status_tags, language_tags, code_size_impact) + ) + + card = CardDirective.create_card( + inst=self, + arguments=[], + options={}, + ) + + # Create the top-level section buttons. + section_buttons = docutils.nodes.bullet_list( + classes=['pw-module-section-buttons'] + ) + + # This is the pattern for required sections. + section_buttons += self.create_section_button( + 'Get Started', self.try_get_option('get-started') + ) + + # This is the pattern for optional sections. + if (tutorials_ref := self.maybe_get_option('tutorials')) is not None: + section_buttons += self.create_section_button( + 'Tutorials', tutorials_ref + ) + + if (guides_ref := self.maybe_get_option('guides')) is not None: + section_buttons += self.create_section_button('Guides', guides_ref) + + if (concepts_ref := self.maybe_get_option('concepts')) is not None: + section_buttons += self.create_section_button( + 'Concepts', concepts_ref + ) + + section_buttons += self.create_section_button( + 'Design', self.try_get_option('design') + ) + + if (api_ref := self.maybe_get_option('api')) is not None: + section_buttons += self.create_section_button( + 'API Reference', api_ref + ) + + return [tagline, section_buttons, content, card] + + +def build_backend_lists(app, _doctree, _fromdocname): + env = app.builder.env + + if not hasattr(env, 'pw_module_relationships'): + env.pw_module_relationships = {} + + +def setup(app: SphinxApplication): + app.add_directive('pigweed-module', PigweedModuleDirective) + + # At this event, the documents and metadata have been generated, and now we + # can modify the doctree to reflect the metadata. + app.connect('doctree-resolved', build_backend_lists) + + return { + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/pw_docgen/py/setup.cfg b/pw_docgen/py/setup.cfg index 3da0a175d..41f7505ef 100644 --- a/pw_docgen/py/setup.cfg +++ b/pw_docgen/py/setup.cfg @@ -22,10 +22,12 @@ description = Generate Sphinx documentation packages = find: zip_safe = False install_requires = - sphinx>3 + sphinx>=5.3.0 sphinx-argparse sphinx-rtd-theme sphinxcontrib-mermaid>=0.7.1 + sphinx-design>=0.3.0 + [options.package_data] pw_docgen = py.typed diff --git a/pw_doctor/py/pw_doctor/doctor.py b/pw_doctor/py/pw_doctor/doctor.py index 427431c64..cf3f8651f 100755 --- a/pw_doctor/py/pw_doctor/doctor.py +++ b/pw_doctor/py/pw_doctor/doctor.py @@ -141,14 +141,16 @@ def pw_plugins(ctx: DoctorContext): ctx.error('Not all pw plugins loaded successfully') -def unames_are_equivalent(uname_actual: str, uname_expected: str) -> bool: +def unames_are_equivalent( + uname_actual: str, uname_expected: str, rosetta: bool = False +) -> bool: """Determine if uname values are equivalent for this tool's purposes.""" # Support `mac-arm64` through Rosetta until `mac-arm64` binaries are ready # Expected and actual unames will not literally match on M1 Macs because # they pretend to be Intel Macs for the purpose of environment setup. But # that's intentional and doesn't require any user action. - if "Darwin" in uname_expected and "arm64" in uname_expected: + if rosetta and "Darwin" in uname_expected and "arm64" in uname_expected: uname_expected = uname_expected.replace("arm64", "x86_64") return uname_actual == uname_expected @@ -178,7 +180,9 @@ def env_os(ctx: DoctorContext): # redundant because it's contained in release or version, and # skipping it here simplifies logic. uname = ' '.join(getattr(os, 'uname', lambda: ())()[2:]) - if not unames_are_equivalent(uname, data['uname']): + rosetta_envvar = os.environ.get('_PW_ROSETTA', '0') + rosetta = rosetta_envvar.strip().lower() != '0' + if not unames_are_equivalent(uname, data['uname'], rosetta): ctx.warning( 'Current uname (%s) does not match Bootstrap uname (%s), ' 'you may need to rerun bootstrap on this system', diff --git a/pw_env_setup/py/BUILD.gn b/pw_env_setup/py/BUILD.gn index 6edd6d61d..7b68fcbda 100644 --- a/pw_env_setup/py/BUILD.gn +++ b/pw_env_setup/py/BUILD.gn @@ -30,6 +30,7 @@ pw_python_package("py") { "pw_env_setup/cipd_setup/update.py", "pw_env_setup/cipd_setup/wrapper.py", "pw_env_setup/colors.py", + "pw_env_setup/config_file.py", "pw_env_setup/env_setup.py", "pw_env_setup/environment.py", "pw_env_setup/gni_visitor.py", diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/cmake.json b/pw_env_setup/py/pw_env_setup/cipd_setup/cmake.json index b90321b23..e200afe69 100644 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/cmake.json +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/cmake.json @@ -13,7 +13,7 @@ "windows-amd64" ], "tags": [ - "version:2@3.25.2.chromium.6" + "version:2@3.26.0.chromium.7" ] } ] diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/go.json b/pw_env_setup/py/pw_env_setup/cipd_setup/go.json index 8c3ad365c..0feb0dbaa 100644 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/go.json +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/go.json @@ -10,7 +10,7 @@ "windows-amd64" ], "tags": [ - "version:2@1.20.1" + "version:2@1.20.2" ] }, { diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/host_tools.json b/pw_env_setup/py/pw_env_setup/cipd_setup/host_tools.json index ab218e161..cf0e8f7bf 100644 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/host_tools.json +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/host_tools.json @@ -8,7 +8,7 @@ "windows-amd64" ], "tags": [ - "git_revision:c28834ee85ac0752bf4d015797f02d88a5ad2cd9" + "git_revision:284a05aeae3cf2e4579e6518ab3a5316058da6d4" ], "version_file": ".versions/host_tools.cipd_version" } diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json b/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json index 7d2d6df38..117ad9712 100644 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/pigweed.json @@ -10,7 +10,7 @@ "windows-amd64" ], "tags": [ - "git_revision:fe330c0ae1ec29db30b6f830e50771a335e071fb" + "git_revision:41fef642de70ecdcaaa26be96d56a0398f95abd4" ], "version_file": ".versions/gn.cipd_version" }, @@ -115,7 +115,7 @@ "mac-amd64" ], "tags": [ - "git_revision:823a3f11fb8f04c3c3cc0f95f968fef1bfc6534f" + "git_revision:823a3f11fb8f04c3c3cc0f95f968fef1bfc6534f,1" ], "version_file": ".versions/qemu.cipd_version" }, diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/python.json b/pw_env_setup/py/pw_env_setup/cipd_setup/python.json index c8e807d34..02dcdfcdb 100644 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/python.json +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/python.json @@ -1,7 +1,4 @@ { - "included_files": [ - "black.json" - ], "packages": [ { "path": "infra/3pp/tools/cpython3/${platform}", diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/update.py b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py index 6a9824461..deac4b0df 100755 --- a/pw_env_setup/py/pw_env_setup/cipd_setup/update.py +++ b/pw_env_setup/py/pw_env_setup/cipd_setup/update.py @@ -133,13 +133,8 @@ def check_auth(cipd, package_files, cipd_service_account, spin): return True -def platform(rosetta=None): +def platform(rosetta=False): """Return the CIPD platform string of the current system.""" - # If running inside a bootstrapped environment we can use the env var. - # Otherwise, require rosetta be set. - if rosetta is None: - rosetta = os.environ['_PW_ROSETTA'] - osname = { 'darwin': 'mac', 'linux': 'linux', diff --git a/pw_env_setup/py/pw_env_setup/config_file.py b/pw_env_setup/py/pw_env_setup/config_file.py new file mode 100644 index 000000000..fc8fa40c5 --- /dev/null +++ b/pw_env_setup/py/pw_env_setup/config_file.py @@ -0,0 +1,43 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +"""Reads and parses the Pigweed config file. + +See also https://pigweed.dev/seed/0101-pigweed.json.html. +""" + +import json +import os + + +def _get_project_root(env=None): + if not env: + env = os.environ + for var in ('PW_PROJECT_ROOT', 'PW_ROOT'): + if var in env: + return env[var] + raise ValueError('environment variable PW_PROJECT_ROOT not set') + + +def path(env=None): + """Return the path where pigweed.json should reside.""" + return os.path.join(_get_project_root(env=env), 'pigweed.json') + + +def load(env=None): + """Load pigweed.json if it exists and return the contents.""" + config_path = path(env=env) + if not os.path.isfile(config_path): + return {} + with open(config_path, 'r') as ins: + return json.load(ins) diff --git a/pw_env_setup/py/pw_env_setup/environment.py b/pw_env_setup/py/pw_env_setup/environment.py index ca7f7f46c..c87de8847 100644 --- a/pw_env_setup/py/pw_env_setup/environment.py +++ b/pw_env_setup/py/pw_env_setup/environment.py @@ -483,6 +483,7 @@ class Environment(object): Yields the new environment object. """ + orig_env = {} try: if export: orig_env = os.environ.copy() diff --git a/pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list b/pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list index 10970a015..bee66aaf3 100644 --- a/pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list +++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list @@ -1,8 +1,9 @@ alabaster==0.7.12 appdirs==1.4.4 -astroid==2.6.6 +astroid==2.14.2 Babel==2.9.1 backcall==0.2.0 +black==23.1.0 build==0.8.0 cachetools==5.0.0 certifi==2021.10.8 @@ -12,6 +13,7 @@ coloredlogs==15.0.1 coverage==6.3 cryptography==36.0.1 decorator==5.1.1 +dill==0.3.6 docutils==0.17.1 breathe==4.34.0 google-api-core==2.7.1 @@ -33,17 +35,19 @@ lazy-object-proxy==1.7.1 MarkupSafe==2.0.1 matplotlib-inline==0.1.3 mccabe==0.6.1 -mypy==0.991 -mypy-extensions==0.4.3 +mypy==1.0.1 +mypy-extensions==1.0.0 mypy-protobuf==3.3.0 -packaging==21.3 +packaging==23.0 parameterized==0.8.1 parso==0.8.3 pep517==0.12.0 pexpect==4.8.0 +platformdirs==3.0.0 pickleshare==0.7.5 prompt-toolkit==3.0.36 -protobuf==3.20.1 +protobuf==3.20.2 +psutil==5.9.4 ptpython==3.0.22 ptyprocess==0.7.0 pyasn1==0.4.8 @@ -51,7 +55,7 @@ pyasn1-modules==0.2.8 pycparser==2.21 pyelftools==0.27 Pygments==2.14.0 -pylint==2.9.3 +pylint==2.16.2 pyparsing==3.0.6 pyperclip==1.8.2 pyserial==3.5 @@ -67,6 +71,7 @@ Sphinx==5.3.0 sphinx-rtd-theme==1.2.0 sphinx-argparse==0.4.0 sphinx-copybutton==0.5.1 +sphinx-design==0.3.0 sphinxcontrib-applehelp==1.0.2 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-htmlhelp==2.0.0 @@ -75,16 +80,17 @@ sphinxcontrib-mermaid==0.8 sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 toml==0.10.2 -tomli==2.0.0 +tomli==2.0.1 +tomlkit==0.11.6 traitlets==5.1.1 types-docutils==0.17.4 types-futures==3.3.2 -types-protobuf==3.19.22 +types-protobuf==3.20.4.6 types-Pygments==2.9.13 types-PyYAML==6.0.7 types-setuptools==63.4.1 types-six==1.16.9 -typing-extensions==4.1.1 +typing-extensions==4.4.0 urllib3==1.26.8 watchdog==2.1.6 wcwidth==0.2.5 diff --git a/pw_env_setup/py/pw_env_setup/virtualenv_setup/pigweed_upstream_requirements.txt b/pw_env_setup/py/pw_env_setup/virtualenv_setup/pigweed_upstream_requirements.txt index 72a71b03c..fcb9baf65 100644 --- a/pw_env_setup/py/pw_env_setup/virtualenv_setup/pigweed_upstream_requirements.txt +++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/pigweed_upstream_requirements.txt @@ -16,9 +16,8 @@ # pigweed.dev Sphinx themes furo==2022.12.7 sphinx-copybutton==0.5.1 -sphinx-design==0.3.0 myst-parser==0.18.1 breathe==4.34.0 # Renode requirements -psutil==5.9.1 +psutil==5.9.4 robotframework==5.0.1 diff --git a/pw_env_setup/util.sh b/pw_env_setup/util.sh index c6e66f1bb..ffcc0c308 100644 --- a/pw_env_setup/util.sh +++ b/pw_env_setup/util.sh @@ -105,7 +105,7 @@ pw_check_root() { pw_error_info " Pigweed's Python environment currently requires Pigweed to" pw_error_info " be at a path without spaces. Please checkout Pigweed in a" pw_error_info " directory without spaces and retry running bootstrap." - return + return -1 fi } diff --git a/pw_function/BUILD.bazel b/pw_function/BUILD.bazel index 6a16156cd..e130d1c87 100644 --- a/pw_function/BUILD.bazel +++ b/pw_function/BUILD.bazel @@ -48,3 +48,34 @@ pw_cc_test( "//pw_compilation_testing:negative_compilation_testing", ], ) + +pw_cc_library( + name = "pointer", + srcs = ["public/pw_function/internal/static_invoker.h"], + hdrs = ["public/pw_function/pointer.h"], + includes = ["public"], +) + +pw_cc_test( + name = "pointer_test", + srcs = ["pointer_test.cc"], + deps = [ + ":pointer", + ":pw_function", + ], +) + +pw_cc_library( + name = "scope_guard", + hdrs = ["public/pw_function/scope_guard.h"], + includes = ["public"], +) + +pw_cc_test( + name = "scope_guard_test", + srcs = ["scope_guard_test.cc"], + deps = [ + ":pw_function", + ":scope_guard", + ], +) diff --git a/pw_function/BUILD.gn b/pw_function/BUILD.gn index cceef9b40..efea1cc8d 100644 --- a/pw_function/BUILD.gn +++ b/pw_function/BUILD.gn @@ -50,6 +50,17 @@ pw_source_set("pw_function") { public = [ "public/pw_function/function.h" ] } +pw_source_set("pointer") { + public_configs = [ ":public_include_path" ] + public = [ "public/pw_function/pointer.h" ] + sources = [ "public/pw_function/internal/static_invoker.h" ] +} + +pw_source_set("scope_guard") { + public_configs = [ ":public_include_path" ] + public = [ "public/pw_function/scope_guard.h" ] +} + pw_doc_group("docs") { sources = [ "docs.rst" ] report_deps = [ @@ -61,6 +72,8 @@ pw_doc_group("docs") { pw_test_group("tests") { tests = [ ":function_test", + ":pointer_test", + ":scope_guard_test", "$dir_pw_third_party/fuchsia:function_tests", ] } @@ -74,6 +87,22 @@ pw_test("function_test") { negative_compilation_tests = true } +pw_test("pointer_test") { + deps = [ + ":pointer", + ":pw_function", + ] + sources = [ "pointer_test.cc" ] +} + +pw_test("scope_guard_test") { + sources = [ "scope_guard_test.cc" ] + deps = [ + ":pw_function", + ":scope_guard", + ] +} + pw_size_diff("function_size") { title = "Pigweed function size report" diff --git a/pw_function/CMakeLists.txt b/pw_function/CMakeLists.txt index bdb0a2d47..8b32db534 100644 --- a/pw_function/CMakeLists.txt +++ b/pw_function/CMakeLists.txt @@ -51,3 +51,40 @@ pw_add_test(pw_function.function_test modules pw_function ) + +pw_add_library(pw_function.pointer INTERFACE + HEADERS + public/pw_function/pointer.h + public/pw_function/internal/static_invoker.h + PUBLIC_INCLUDES + public +) + +pw_add_test(pw_function.pointer_test + SOURCES + pointer_test.cc + PRIVATE_DEPS + pw_function + pw_function.pointer + GROUPS + modules + pw_function +) + +pw_add_library(pw_function.scope_guard INTERFACE + HEADERS + public/pw_function/scope_guard.h + PUBLIC_INCLUDES + public +) + +pw_add_test(pw_function.scope_guard_test + SOURCES + scope_guard_test.cc + PRIVATE_DEPS + pw_function + pw_function.scope_guard + GROUPS + modules + pw_function +) diff --git a/pw_function/docs.rst b/pw_function/docs.rst index 92ec7afa8..18041ef51 100644 --- a/pw_function/docs.rst +++ b/pw_function/docs.rst @@ -1,19 +1,20 @@ .. _module-pw_function: ------------ +=========== pw_function ------------ +=========== The ``pw_function`` module provides a standard, general-purpose API for wrapping callable objects. ``pw_function`` is similar in spirit and API to ``std::function``, but doesn't allocate, and uses several tricks to prevent code bloat. +-------- Overview -======== +-------- Basic usage ------------ -``pw_function`` defines the ``pw::Function`` class. A ``Function`` is a +=========== +``pw_function`` defines the :cpp:type:`pw::Function` class. A ``Function`` is a move-only callable wrapper constructable from any callable object. Functions are templated on the signature of the callable they store. @@ -53,9 +54,9 @@ Functions are nullable. Invoking a null function triggers a runtime assert. function(); } -``pw::Function``'s default constructor is ``constexpr``, so default-constructed -functions may be used in classes with ``constexpr`` constructors and in -``constinit`` expressions. +:cpp:type:`pw::Function`'s default constructor is ``constexpr``, so +default-constructed functions may be used in classes with ``constexpr`` +constructors and in ``constinit`` expressions. .. code-block:: c++ @@ -71,16 +72,16 @@ functions may be used in classes with ``constexpr`` constructors and in constinit MyClass instance; Storage -------- +======= By default, a ``Function`` stores its callable inline within the object. The inline storage size defaults to the size of two pointers, but is configurable through the build system. The size of a ``Function`` object is equivalent to its inline storage size. -The ``pw::InlineFunction`` alias is similar to ``pw::Function``, but is always -inlined. That is, even if dynamic allocation is enabled for ``pw::Function`` - -``pw::InlineFunction`` will fail to compile if the callable is larger than the -inline storage size. +The :cpp:type:`pw::InlineFunction` alias is similar to :cpp:type:`pw::Function`, +but is always inlined. That is, even if dynamic allocation is enabled for +:cpp:type:`pw::Function`, :cpp:type:`pw::InlineFunction` will fail to compile if +the callable is larger than the inline storage size. Attempting to construct a function from a callable larger than its inline size is a compile-time error unless dynamic allocation is enabled. @@ -117,15 +118,23 @@ is a compile-time error unless dynamic allocation is enabled. ``pw::InlineFunction`` can be used. .. warning:: - If ``PW_FUNCTION_ENABLE_DYNAMIC_ALLOCATION`` is enabled then attempt to cast - from ``pw::InlineFunction`` to a regular ``pw::Function`` will **ALWAYS** - allocate memory. + If ``PW_FUNCTION_ENABLE_DYNAMIC_ALLOCATION`` is enabled then attempts to cast + from `:cpp:type:`pw::InlineFunction` to a regular :cpp:type:`pw::Function` + will **ALWAYS** allocate memory. +--------- API usage +--------- + +Reference ========= +.. doxygentypedef:: pw::Function +.. doxygentypedef:: pw::InlineFunction +.. doxygentypedef:: pw::Callback +.. doxygentypedef:: pw::InlineCallback -``pw::Function`` function parameters ------------------------------------- +``pw::Function`` as a function parameter +======================================== When implementing an API which takes a callback, a ``Function`` can be used in place of a function pointer or equivalent callable. @@ -138,12 +147,13 @@ place of a function pointer or equivalent callable. // signature template for clarity. void DoTheThing(int arg, const pw::Function<void(int result)>& callback); -``pw::Function`` is movable, but not copyable, so APIs must accept -``pw::Function`` objects either by const reference (``const +:cpp:type:`pw::Function` is movable, but not copyable, so APIs must accept +:cpp:type:`pw::Function` objects either by const reference (``const pw::Function<void()>&``) or rvalue reference (``const pw::Function<void()>&&``). -If the ``pw::Function`` simply needs to be called, it should be passed by const -reference. If the ``pw::Function`` needs to be stored, it should be passed as an -rvalue reference and moved into a ``pw::Function`` variable as appropriate. +If the :cpp:type:`pw::Function` simply needs to be called, it should be passed +by const reference. If the :cpp:type:`pw::Function` needs to be stored, it +should be passed as an rvalue reference and moved into a +:cpp:type:`pw::Function` variable as appropriate. .. code-block:: c++ @@ -159,38 +169,40 @@ rvalue reference and moved into a ``pw::Function`` variable as appropriate. stored_callback_ = std::move(callback); } -.. admonition:: Rules of thumb for passing a ``pw::Function`` to a function +.. admonition:: Rules of thumb for passing a :cpp:type:`pw::Function` to a function * **Pass by value**: Never. - This results in unnecessary ``pw::Function`` instances and move operations. + This results in unnecessary :cpp:type:`pw::Function` instances and move + operations. * **Pass by const reference** (``const pw::Function&``): When the - ``pw::Function`` is only invoked. + :cpp:type:`pw::Function` is only invoked. - When a ``pw::Function`` is called or inspected, but not moved, take a const - reference to avoid copies and support temporaries. + When a :cpp:type:`pw::Function` is called or inspected, but not moved, take + a const reference to avoid copies and support temporaries. * **Pass by rvalue reference** (``pw::Function&&``): When the - ``pw::Function`` is moved. - - When the function takes ownership of the ``pw::Function`` object, always - use an rvalue reference (``pw::Function<void()>&&``) instead of a mutable - lvalue reference (``pw::Function<void()>&``). An rvalue reference forces - the caller to ``std::move`` when passing a preexisting ``pw::Function`` - variable, which makes the transfer of ownership explicit. It is possible to - move-assign from an lvalue reference, but this fails to make it obvious to - the caller that the object is no longer valid. + :cpp:type:`pw::Function` is moved. + + When the function takes ownership of the :cpp:type:`pw::Function` object, + always use an rvalue reference (``pw::Function<void()>&&``) instead of a + mutable lvalue reference (``pw::Function<void()>&``). An rvalue reference + forces the caller to ``std::move`` when passing a preexisting + :cpp:type:`pw::Function` variable, which makes the transfer of ownership + explicit. It is possible to move-assign from an lvalue reference, but this + fails to make it obvious to the caller that the object is no longer valid. * **Pass by non-const reference** (``pw::Function&``): Rarely, when modifying a variable. Non-const references are only necessary when modifying an existing - ``pw::Function`` variable. Use an rvalue reference instead if the - ``pw::Function`` is moved into another variable. + :cpp:type:`pw::Function` variable. Use an rvalue reference instead if the + :cpp:type:`pw::Function` is moved into another variable. Calling functions that use ``pw::Function`` -------------------------------------------- -A ``pw::Function`` can be implicitly constructed from any callback object. When -calling an API that takes a ``pw::Function``, simply pass the callable object. -There is no need to create an intermediate ``pw::Function`` object. +=========================================== +A :cpp:type:`pw::Function` can be implicitly constructed from any callback +object. When calling an API that takes a :cpp:type:`pw::Function`, simply pass +the callable object. There is no need to create an intermediate +:cpp:type:`pw::Function` object. .. code-block:: c++ @@ -200,10 +212,10 @@ There is no need to create an intermediate ``pw::Function`` object. // Implicitly creates a pw::Function from a capturing lambda and stores it. StoreTheCallback([this](int result) { result_ = result; }); -When working with an existing ``pw::Function`` variable, the variable can be -passed directly to functions that take a const reference. If the function takes -ownership of the ``pw::Function``, move the ``pw::Function`` variable at the -call site. +When working with an existing :cpp:type:`pw::Function` variable, the variable +can be passed directly to functions that take a const reference. If the function +takes ownership of the :cpp:type:`pw::Function`, move the +:cpp:type:`pw::Function` variable at the call site. .. code-block:: c++ @@ -213,42 +225,59 @@ call site. // Takes ownership of the pw::Function. void StoreTheCallback(std::move(my_function)); -Use ``pw::Callback`` for one-shot functions -------------------------------------------- -``pw::Callback`` is a specialization of ``pw::Function`` that can only be called -once. After a ``pw::Callback`` is called, the target function is destroyed. A -``pw::Callback`` in the "already called" state has the same state as a -``pw::Callback`` that has been assigned to nullptr. - +``pw::Callback`` for one-shot functions +======================================= +:cpp:type:`pw::Callback` is a specialization of :cpp:type:`pw::Function` that +can only be called once. After a :cpp:type:`pw::Callback` is called, the target +function is destroyed. A :cpp:type:`pw::Callback` in the "already called" state +has the same state as a :cpp:type:`pw::Callback` that has been assigned to +nullptr. + +Invoking ``pw::Function`` from a C-style API +============================================ +.. doxygenfile:: pw_function/pointer.h + :sections: detaileddescription + +.. doxygenfunction:: GetFunctionPointer() +.. doxygenfunction:: GetFunctionPointer(const FunctionType&) +.. doxygenfunction:: GetFunctionPointerContextFirst() +.. doxygenfunction:: GetFunctionPointerContextFirst(const FunctionType&) + +---------- +ScopeGuard +---------- +.. doxygenclass:: pw::ScopeGuard + :members: + +------------ Size reports -============ +------------ Function class --------------- -The following size report compares an API using a ``pw::Function`` to a +============== +The following size report compares an API using a :cpp:type:`pw::Function` to a traditional function pointer. .. include:: function_size Callable sizes --------------- +============== The table below demonstrates typical sizes of various callable types, which can be used as a reference when sizing external buffers for ``Function`` objects. .. include:: callable_size +------ Design -====== -``pw::Function`` is an alias of -`fit::function <https://cs.opensource.google/fuchsia/fuchsia/+/main:sdk/lib/fit/include/lib/fit/function.h;drc=f66f54fca0c11a1168d790bcc3d8a5a3d940218d>`_ -. - +------ +:cpp:type:`pw::Function` is an alias of +`fit::function <https://cs.opensource.google/fuchsia/fuchsia/+/main:sdk/lib/fit/include/lib/fit/function.h;drc=f66f54fca0c11a1168d790bcc3d8a5a3d940218d>`_. -``pw::Callback`` is an alias of -`fit::callback <https://cs.opensource.google/fuchsia/fuchsia/+/main:sdk/lib/fit/include/lib/fit/function.h;drc=f66f54fca0c11a1168d790bcc3d8a5a3d940218d>`_ -. +:cpp:type:`pw::Callback` is an alias of +`fit::callback <https://cs.opensource.google/fuchsia/fuchsia/+/main:sdk/lib/fit/include/lib/fit/function.h;drc=f66f54fca0c11a1168d790bcc3d8a5a3d940218d>`_. +------ Zephyr -====== +------ To enable ``pw_function` for Zephyr add ``CONFIG_PIGWEED_FUNCTION=y`` to the project's configuration. diff --git a/pw_function/pointer_test.cc b/pw_function/pointer_test.cc new file mode 100644 index 000000000..4951db219 --- /dev/null +++ b/pw_function/pointer_test.cc @@ -0,0 +1,126 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#include "pw_function/pointer.h" + +#include "gtest/gtest.h" +#include "pw_function/function.h" + +namespace pw::function { +namespace { + +extern "C" void pw_function_test_InvokeFromCApi(void (*function)(void* context), + void* context) { + function(context); +} + +extern "C" int pw_function_test_Sum(int (*summer)(int a, int b, void* context), + void* context) { + return summer(60, 40, context); +} + +TEST(StaticInvoker, Function_NoArguments) { + int value = 0; + Function<void()> set_value_to_42([&value] { value += 42; }); + + pw_function_test_InvokeFromCApi(GetFunctionPointer(set_value_to_42), + &set_value_to_42); + + EXPECT_EQ(value, 42); + + pw_function_test_InvokeFromCApi( + GetFunctionPointer<decltype(set_value_to_42)>(), &set_value_to_42); + + EXPECT_EQ(value, 84); +} + +TEST(StaticInvoker, Function_WithArguments) { + int sum = 0; + Function<int(int, int)> sum_stuff([&sum](int a, int b) { + sum += a + b; + return a + b; + }); + + EXPECT_EQ(100, + pw_function_test_Sum(GetFunctionPointer(sum_stuff), &sum_stuff)); + + EXPECT_EQ(sum, 100); + + EXPECT_EQ(100, + pw_function_test_Sum(GetFunctionPointer<decltype(sum_stuff)>(), + &sum_stuff)); + + EXPECT_EQ(sum, 200); +} + +TEST(StaticInvoker, Callback_NoArguments) { + int value = 0; + Callback<void()> set_value_to_42([&value] { value += 42; }); + + pw_function_test_InvokeFromCApi(GetFunctionPointer(set_value_to_42), + &set_value_to_42); + + EXPECT_EQ(value, 42); +} + +TEST(StaticInvoker, Callback_WithArguments) { + int sum = 0; + Callback<int(int, int)> sum_stuff([&sum](int a, int b) { + sum += a + b; + return a + b; + }); + + EXPECT_EQ(100, + pw_function_test_Sum(GetFunctionPointer<decltype(sum_stuff)>(), + &sum_stuff)); + + EXPECT_EQ(sum, 100); +} + +TEST(StaticInvoker, Lambda_NoArguments) { + int value = 0; + auto set_value_to_42([&value] { value += 42; }); + + pw_function_test_InvokeFromCApi(GetFunctionPointer(set_value_to_42), + &set_value_to_42); + + EXPECT_EQ(value, 42); + + pw_function_test_InvokeFromCApi( + GetFunctionPointer<decltype(set_value_to_42)>(), &set_value_to_42); + + EXPECT_EQ(value, 84); +} + +TEST(StaticInvoker, Lambda_WithArguments) { + int sum = 0; + auto sum_stuff = [&sum](int a, int b) { + sum += a + b; + return a + b; + }; + + EXPECT_EQ(100, + pw_function_test_Sum(GetFunctionPointer(sum_stuff), &sum_stuff)); + + EXPECT_EQ(sum, 100); + + EXPECT_EQ(100, + pw_function_test_Sum(GetFunctionPointer<decltype(sum_stuff)>(), + &sum_stuff)); + + EXPECT_EQ(sum, 200); +} + +} // namespace +} // namespace pw::function diff --git a/pw_function/public/pw_function/function.h b/pw_function/public/pw_function/function.h index 736e14c10..99842b348 100644 --- a/pw_function/public/pw_function/function.h +++ b/pw_function/public/pw_function/function.h @@ -18,33 +18,34 @@ namespace pw { -// pw::Function is a wrapper for an arbitrary callable object. It can be used by -// callback-based APIs to allow callers to provide any type of callable. -// -// Example: -// -// template <typename T> -// bool All(const pw::Vector<T>& items, -// pw::Function<bool(const T& item)> predicate) { -// for (const T& item : items) { -// if (!predicate(item)) { -// return false; -// } -// } -// return true; -// } -// -// bool ElementsArePositive(const pw::Vector<int>& items) { -// return All(items, [](const int& i) { return i > 0; }); -// } -// -// bool IsEven(const int& i) { return i % 2 == 0; } -// -// bool ElementsAreEven(const pw::Vector<int>& items) { -// return All(items, IsEven); -// } -// - +/// `pw::Function` is a wrapper for an arbitrary callable object. It can be used +/// by callback-based APIs to allow callers to provide any type of callable. +/// +/// Example: +/// @code{.cpp} +/// +/// template <typename T> +/// bool All(const pw::Vector<T>& items, +/// pw::Function<bool(const T& item)> predicate) { +/// for (const T& item : items) { +/// if (!predicate(item)) { +/// return false; +/// } +/// } +/// return true; +/// } +/// +/// bool ElementsArePositive(const pw::Vector<int>& items) { +/// return All(items, [](const int& i) { return i > 0; }); +/// } +/// +/// bool IsEven(const int& i) { return i % 2 == 0; } +/// +/// bool ElementsAreEven(const pw::Vector<int>& items) { +/// return All(items, IsEven); +/// } +/// +/// @endcode template <typename Callable, size_t inline_target_size = function_internal::config::kInlineCallableSize> @@ -53,14 +54,14 @@ using Function = fit::function_impl< /*require_inline=*/!function_internal::config::kEnableDynamicAllocation, Callable>; -// Always inlined version of pw::Function. -// -// IMPORTANT: If pw::Function is configured to allow dynamic allocations then -// any attempt to convert `pw::InlineFunction` to `pw::Function` will ALWAYS -// allocate. -// +/// Version of `pw::Function` that exclusively uses inline storage. +/// +/// IMPORTANT: If `pw::Function` is configured to allow dynamic allocations then +/// any attempt to convert `pw::InlineFunction` to `pw::Function` will ALWAYS +/// allocate. +/// // TODO(b/252852651): Remove warning above when conversion from -// fit::inline_function to fit::function doesn't allocate anymore. +// `fit::inline_function` to `fit::function` doesn't allocate anymore. template <typename Callable, size_t inline_target_size = function_internal::config::kInlineCallableSize> @@ -68,16 +69,16 @@ using InlineFunction = fit::inline_function<Callable, inline_target_size>; using Closure = Function<void()>; -// pw::Callback is identical to pw::Function except: -// -// 1) On the first call to invoke a `pw::Callback`, the target function held -// by the `pw::Callback` cannot be called again. -// 2) When a `pw::Callback` is invoked for the first time, the target function -// is released and destructed, along with any resources owned by that -// function (typically the objects captured by a lambda). - -// A `pw::Callback` in the "already called" state has the same state as a -// `pw::Callback` that has been assigned to `nullptr`. +/// `pw::Callback` is identical to @cpp_type{pw::Function} except: +/// +/// 1. On the first call to invoke a `pw::Callback`, the target function held +/// by the `pw::Callback` cannot be called again. +/// 2. When a `pw::Callback` is invoked for the first time, the target function +/// is released and destructed, along with any resources owned by that +/// function (typically the objects captured by a lambda). +/// +/// A `pw::Callback` in the "already called" state has the same state as a +/// `pw::Callback` that has been assigned to `nullptr`. template <typename Callable, size_t inline_target_size = function_internal::config::kInlineCallableSize> @@ -86,6 +87,7 @@ using Callback = fit::callback_impl< /*require_inline=*/!function_internal::config::kEnableDynamicAllocation, Callable>; +/// Version of `pw::Callback` that exclusively uses inline storage. template <typename Callable, size_t inline_target_size = function_internal::config::kInlineCallableSize> diff --git a/pw_function/public/pw_function/internal/static_invoker.h b/pw_function/public/pw_function/internal/static_invoker.h new file mode 100644 index 000000000..73605af34 --- /dev/null +++ b/pw_function/public/pw_function/internal/static_invoker.h @@ -0,0 +1,43 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#pragma once + +// This code is in its own header since Doxygen / Breathe can't parse it. + +namespace pw::function::internal { + +template <typename FunctionType, typename CallOperatorType> +struct StaticInvoker; + +template <typename FunctionType, typename Return, typename... Args> +struct StaticInvoker<FunctionType, Return (FunctionType::*)(Args...)> { + // Invoker function with the context argument last to match libc. Could add a + // version with the context first if needed. + static Return InvokeWithContextLast(Args... args, void* context) { + return static_cast<FunctionType*>(context)->operator()( + std::forward<Args>(args)...); + } + + static Return InvokeWithContextFirst(void* context, Args... args) { + return static_cast<FunctionType*>(context)->operator()( + std::forward<Args>(args)...); + } +}; + +// Make the const version identical to the non-const version. +template <typename FunctionType, typename Return, typename... Args> +struct StaticInvoker<FunctionType, Return (FunctionType::*)(Args...) const> + : StaticInvoker<FunctionType, Return (FunctionType::*)(Args...)> {}; + +} // namespace pw::function::internal diff --git a/pw_function/public/pw_function/pointer.h b/pw_function/public/pw_function/pointer.h new file mode 100644 index 000000000..8915eb062 --- /dev/null +++ b/pw_function/public/pw_function/pointer.h @@ -0,0 +1,97 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#pragma once + +/// @file pw_function/pointer.h +/// +/// Traditional callback APIs often use a function pointer and `void*` context +/// argument. The context argument makes it possible to use the callback +/// function with non-global data. For example, the `qsort_s` and `bsearch_s` +/// functions take a pointer to a comparison function that has `void*` context +/// as its last parameter. @cpp_type{pw::Function} does not naturally work with +/// these kinds of APIs. +/// +/// The functions below make it simple to adapt a @cpp_type{pw::Function} for +/// use with APIs that accept a function pointer and `void*` context argument. + +#include <utility> + +#include "pw_function/internal/static_invoker.h" + +namespace pw::function { + +/// Returns a function pointer that invokes a `pw::Function`, lambda, or other +/// callable object from a `void*` context argument. This makes it possible to +/// use C++ callables with C-style APIs that take a function pointer and `void*` +/// context. +/// +/// The returned function pointer has the same return type and arguments as the +/// `pw::Function` or `pw::Callback`, except that the last parameter is a +/// `void*`. `GetFunctionPointerContextFirst` places the `void*` context +/// parameter first. +/// +/// The following example adapts a C++ lambda function for use with C-style API +/// that takes an `int (*)(int, void*)` function and a `void*` context. +/// +/// @code{.cpp} +/// +/// void TakesAFunctionPointer(int (*function)(int, void*), void* context); +/// +/// void UseFunctionPointerApiWithPwFunction() { +/// // Declare a callable object so a void* pointer can be obtained for it. +/// auto my_function = [captures](int value) { +/// // ... +/// return value + captures; +/// }; +/// +/// // Invoke the API with the function pointer and callable pointer. +/// TakesAFunctionPointer(pw::function::GetFunctionPointer(my_function), +/// &my_function); +/// } +/// +/// @endcode +/// +/// The function returned from this must ONLY be used with the exact type for +/// which it was created! Function pointer / context APIs are not type safe. +template <typename FunctionType> +constexpr auto GetFunctionPointer() { + return internal::StaticInvoker< + FunctionType, + decltype(&FunctionType::operator())>::InvokeWithContextLast; +} + +/// `GetFunctionPointer` overload that uses the type of the function passed to +/// this call. +template <typename FunctionType> +constexpr auto GetFunctionPointer(const FunctionType&) { + return GetFunctionPointer<FunctionType>(); +} + +/// Same as `GetFunctionPointer`, but the context argument is passed first. +/// Returns a `void(void*, int)` function for a `pw::Function<void(int)>`. +template <typename FunctionType> +constexpr auto GetFunctionPointerContextFirst() { + return internal::StaticInvoker< + FunctionType, + decltype(&FunctionType::operator())>::InvokeWithContextFirst; +} + +/// `GetFunctionPointerContextFirst` overload that uses the type of the function +/// passed to this call. +template <typename FunctionType> +constexpr auto GetFunctionPointerContextFirst(const FunctionType&) { + return GetFunctionPointerContextFirst<FunctionType>(); +} + +} // namespace pw::function diff --git a/pw_function/public/pw_function/scope_guard.h b/pw_function/public/pw_function/scope_guard.h new file mode 100644 index 000000000..ea790ad72 --- /dev/null +++ b/pw_function/public/pw_function/scope_guard.h @@ -0,0 +1,87 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#pragma once + +#include <utility> + +namespace pw { + +/// `ScopeGuard` ensures that the specified functor is executed no matter how +/// the current scope exits, unless it is dismissed. +/// +/// Example: +/// +/// @code +/// pw::Status SomeFunction() { +/// PW_TRY(OperationOne()); +/// ScopeGuard undo_operation_one(UndoOperationOne); +/// PW_TRY(OperationTwo()); +/// ScopeGuard undo_operation_two(UndoOperationTwo); +/// PW_TRY(OperationThree()); +/// undo_operation_one.Dismiss(); +/// undo_operation_two.Dismiss(); +/// return pw::OkStatus(); +/// } +/// @endcode +template <typename Functor> +class ScopeGuard { + public: + constexpr ScopeGuard(Functor&& functor) + : functor_(std::forward<Functor>(functor)), dismissed_(false) {} + + constexpr ScopeGuard(ScopeGuard&& other) noexcept + : functor_(std::move(other.functor_)), dismissed_(other.dismissed_) { + other.dismissed_ = true; + } + + template <typename OtherFunctor> + constexpr ScopeGuard(ScopeGuard<OtherFunctor>&& other) + : functor_(std::move(other.functor_)), dismissed_(other.dismissed_) { + other.dismissed_ = true; + } + + ~ScopeGuard() { + if (dismissed_) { + return; + } + functor_(); + } + + ScopeGuard& operator=(ScopeGuard&& other) noexcept { + functor_ = std::move(other.functor_); + dismissed_ = std::move(other.dismissed_); + other.dismissed_ = true; + } + + ScopeGuard() = delete; + ScopeGuard(const ScopeGuard&) = delete; + ScopeGuard& operator=(const ScopeGuard&) = delete; + + /// Dismisses the `ScopeGuard`, meaning it will no longer execute the + /// `Functor` when it goes out of scope. + void Dismiss() { dismissed_ = true; } + + private: + template <typename OtherFunctor> + friend class ScopeGuard; + + Functor functor_; + bool dismissed_; +}; + +// Enable type deduction for a compatible function pointer. +template <typename Function> +ScopeGuard(Function()) -> ScopeGuard<Function (*)()>; + +} // namespace pw diff --git a/pw_function/scope_guard_test.cc b/pw_function/scope_guard_test.cc new file mode 100644 index 000000000..404293a1f --- /dev/null +++ b/pw_function/scope_guard_test.cc @@ -0,0 +1,88 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#include "pw_function/scope_guard.h" + +#include <utility> + +#include "gtest/gtest.h" +#include "pw_function/function.h" + +namespace pw { +namespace { + +TEST(ScopeGuard, ExecutesLambda) { + bool executed = false; + { + ScopeGuard guarded_lambda([&] { executed = true; }); + EXPECT_FALSE(executed); + } + EXPECT_TRUE(executed); +} + +static bool static_executed = false; +void set_static_executed() { static_executed = true; } + +TEST(ScopeGuard, ExecutesFunction) { + { + ScopeGuard guarded_function(set_static_executed); + + EXPECT_FALSE(static_executed); + } + EXPECT_TRUE(static_executed); +} + +TEST(ScopeGuard, ExecutesPwFunction) { + bool executed = false; + pw::Function<void()> pw_function([&]() { executed = true; }); + { + ScopeGuard guarded_pw_function(std::move(pw_function)); + EXPECT_FALSE(executed); + } + EXPECT_TRUE(executed); +} + +TEST(ScopeGuard, Dismiss) { + bool executed = false; + { + ScopeGuard guard([&] { executed = true; }); + EXPECT_FALSE(executed); + guard.Dismiss(); + EXPECT_FALSE(executed); + } + EXPECT_FALSE(executed); +} + +TEST(ScopeGuard, MoveConstructor) { + bool executed = false; + ScopeGuard first_guard([&] { executed = true; }); + { + ScopeGuard second_guard(std::move(first_guard)); + EXPECT_FALSE(executed); + } + EXPECT_TRUE(executed); +} + +TEST(ScopeGuard, MoveOperator) { + bool executed = false; + ScopeGuard first_guard([&] { executed = true; }); + { + ScopeGuard second_guard = std::move(first_guard); + EXPECT_FALSE(executed); + } + EXPECT_TRUE(executed); +} + +} // namespace +} // namespace pw diff --git a/pw_fuzzer/BUILD.gn b/pw_fuzzer/BUILD.gn index cef173d00..142b7617d 100644 --- a/pw_fuzzer/BUILD.gn +++ b/pw_fuzzer/BUILD.gn @@ -17,32 +17,39 @@ import("//build_overrides/pigweed.gni") import("$dir_pw_build/target_types.gni") import("$dir_pw_docgen/docs.gni") import("$dir_pw_fuzzer/fuzzer.gni") -import("$dir_pw_fuzzer/oss_fuzz.gni") config("public_include_path") { include_dirs = [ "public" ] visibility = [ ":*" ] } -# This is added automatically by the `pw_fuzzer` template. -config("fuzzing") { - common_flags = [ "-fsanitize=fuzzer" ] - cflags = common_flags - ldflags = common_flags -} +# Add flags for adding LLVM sanitizer coverage for fuzzing. This is added by +# the host_clang_fuzz toolchains. +config("instrumentation") { + if (pw_toolchain_OSS_FUZZ_ENABLED) { + # OSS-Fuzz manipulates compiler flags directly. See + # google.github.io/oss-fuzz/getting-started/new-project-guide/#Requirements. + cflags_c = string_split(getenv("CFLAGS")) + cflags_cc = string_split(getenv("CXXFLAGS")) -# OSS-Fuzz needs to be able to specify its own compilers and add flags. -config("oss_fuzz") { - # OSS-Fuzz doesn't always link with -fsanitize=fuzzer, sometimes it uses - #-fsanitize=fuzzer-no-link and provides the fuzzing engine explicitly to be - # passed to the linker. - ldflags = [ getenv("LIB_FUZZING_ENGINE") ] + # OSS-Fuzz sets "-stdlib=libc++", which conflicts with the "-nostdinc++" set + # by `pw_minimal_cpp_stdlib`. + cflags_cc += [ "-Wno-unused-command-line-argument" ] + } else { + cflags = [ "-fsanitize=fuzzer-no-link" ] + } } -config("oss_fuzz_extra") { - cflags_c = oss_fuzz_extra_cflags_c - cflags_cc = oss_fuzz_extra_cflags_cc - ldflags = oss_fuzz_extra_ldflags +# Add flags for linking against compiler-rt's libFuzzer. This is added +# automatically by `pw_fuzzer`. +config("engine") { + if (pw_toolchain_OSS_FUZZ_ENABLED) { + # OSS-Fuzz manipulates linker flags directly. See + # google.github.io/oss-fuzz/getting-started/new-project-guide/#Requirements. + ldflags = string_split(getenv("LDFLAGS")) + [ getenv("LIB_FUZZING_ENGINE") ] + } else { + ldflags = [ "-fsanitize=fuzzer" ] + } } pw_source_set("pw_fuzzer") { @@ -81,12 +88,15 @@ pw_doc_group("docs") { pw_fuzzer("toy_fuzzer") { sources = [ "examples/toy_fuzzer.cc" ] deps = [ - dir_pw_result, - dir_pw_span, - dir_pw_string, + ":pw_fuzzer", + dir_pw_status, ] } pw_test_group("tests") { - tests = [ ":toy_fuzzer" ] + tests = [ ":toy_fuzzer_test" ] +} + +group("fuzzers") { + deps = [ ":toy_fuzzer" ] } diff --git a/pw_fuzzer/docs.rst b/pw_fuzzer/docs.rst index e52b9f163..02bf05f45 100644 --- a/pw_fuzzer/docs.rst +++ b/pw_fuzzer/docs.rst @@ -68,8 +68,9 @@ Building fuzzers with GN To build a fuzzer, do the following: -1. Add the GN target using ``pw_fuzzer`` GN template, and add it to your the - test group of the module: +1. Add the GN target to the module using ``pw_fuzzer`` GN template. If you wish + to limit when the generated unit test is run, you can set `enable_test_if` in + the same manner as `enable_if` for `pw_test`: .. code:: @@ -79,25 +80,65 @@ To build a fuzzer, do the following: pw_fuzzer("my_fuzzer") { sources = [ "my_fuzzer.cc" ] deps = [ ":my_lib" ] + enable_test_if = device_has_1m_flash } +2. Add the generated unit test to the module's test group. This test verifies + the fuzzer can build and run, even when not being built in a fuzzing + toolchain. + +.. code:: + + # In $dir_my_module/BUILD.gn pw_test_group("tests") { tests = [ - ":existing_tests", ... - ":my_fuzzer", # <- Added! + ... + ":my_fuzzer_test", + ] + } + +3. If your module does not already have a group of fuzzers, add it and include + it in the top level fuzzers target. Depending on your project, the specific + toolchain may differ. Fuzzer toolchains are those with + ``pw_toolchain_FUZZING_ENABLED`` set to true. Examples include + ``host_clang_fuzz`` and any toolchains that extend it. + +.. code:: + + # In //BUILD.gn + group("fuzzers") { + deps = [ + ... + "$dir_my_module:fuzzers($dir_pigweed/targets/host:host_clang_fuzz)", + ] + } + +4. Add your fuzzer to the module's group of fuzzers. + +.. code:: + + group("fuzzers") { + deps = [ + ... + ":my_fuzzer", ] } -2. Select your choice of sanitizers ("address" is also the current default). +5. If desired, select a sanitizer runtime. By default, + `//targets/host:host_clang_fuzz` uses "address" if no sanitizer is specified. See LLVM for `valid options`_. .. code:: sh $ gn gen out --args='pw_toolchain_SANITIZERS=["address"]' -3. Build normally, e.g. using ``pw watch``. +6. Build the fuzzers! -.. _run: +.. code:: sh + + $ ninja -C out fuzzers + +.. _bazel: Building and running fuzzers with Bazel ======================================= @@ -142,6 +183,8 @@ To build a fuzzer, do the following: bazel test //my_module:my_fuzz_test --config asan-libfuzzer +.. _run: + Running fuzzers locally ======================= diff --git a/pw_fuzzer/examples/toy_fuzzer.cc b/pw_fuzzer/examples/toy_fuzzer.cc index 2576e2a71..99b04df67 100644 --- a/pw_fuzzer/examples/toy_fuzzer.cc +++ b/pw_fuzzer/examples/toy_fuzzer.cc @@ -21,70 +21,32 @@ #include <cstddef> #include <cstdint> -#include <cstring> +#include <string_view> -#include "pw_result/result.h" -#include "pw_span/span.h" -#include "pw_string/util.h" +#include "pw_fuzzer/fuzzed_data_provider.h" +#include "pw_status/status.h" +namespace pw::fuzzer::example { namespace { // The code to fuzz. This would normally be in separate library. -void toy_example(const char* word1, const char* word2) { - bool greeted = false; - if (word1[0] == 'h') { - if (word1[1] == 'e') { - if (word1[2] == 'l') { - if (word1[3] == 'l') { - if (word1[4] == 'o') { - greeted = true; - } - } - } - } - } - if (word2[0] == 'w') { - if (word2[1] == 'o') { - if (word2[2] == 'r') { - if (word2[3] == 'l') { - if (word2[4] == 'd') { - if (greeted) { - // Our "defect", simulating a crash. - __builtin_trap(); - } - } - } - } +Status SomeAPI(std::string_view s1, std::string_view s2) { + if (s1 == "hello") { + if (s2 == "world") { + abort(); } } + return OkStatus(); } } // namespace +} // namespace pw::fuzzer::example // The fuzz target function extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - // We want to split our input into two strings. - const pw::span<const char> input(reinterpret_cast<const char*>(data), size); - - // If that's not feasible, toss this input. The fuzzer will quickly learn that - // inputs without null-terminators are uninteresting. - const pw::Result<size_t> possible_word1_size = - pw::string::NullTerminatedLength(input); - if (!possible_word1_size.ok()) { - return 0; - } - const pw::span<const char> word1 = - input.first(possible_word1_size.value() + 1); - - // Actually, inputs without TWO null terminators are uninteresting. - pw::span<const char> remaining_input = input.subspan(word1.size()); - if (!pw::string::NullTerminatedLength(remaining_input).ok()) { - return 0; - } - - // Call the code we're targeting! - toy_example(word1.data(), remaining_input.data()); - - // By convention, the fuzzer always returns zero. + FuzzedDataProvider provider(data, size); + std::string s1 = provider.ConsumeRandomLengthString(); + std::string s2 = provider.ConsumeRemainingBytesAsString(); + pw::fuzzer::example::SomeAPI(s1, s2).IgnoreError(); return 0; } diff --git a/pw_fuzzer/fuzzer.gni b/pw_fuzzer/fuzzer.gni index beb3a1c5d..a43a2d9cb 100644 --- a/pw_fuzzer/fuzzer.gni +++ b/pw_fuzzer/fuzzer.gni @@ -14,10 +14,11 @@ import("//build_overrides/pigweed.gni") +import("$dir_pw_build/error.gni") import("$dir_pw_toolchain/host_clang/toolchains.gni") import("$dir_pw_unit_test/test.gni") -# Creates a libFuzzer-based fuzzer executable target. +# Creates a libFuzzer-based fuzzer executable target and unit test # # This will link `sources` and `deps` with the libFuzzer compiler runtime. The # `sources` and `deps` should include a definition of the standard LLVM fuzz @@ -25,82 +26,47 @@ import("$dir_pw_unit_test/test.gni") # //pw_fuzzer/docs.rst # https://llvm.org/docs/LibFuzzer.html # +# Additionally, this creates a unit test that does not generate fuzzer inputs +# and simply executes the fuzz target function with fixed inputs. This is useful +# for verifying the fuzz target function compiles, links, and runs even when not +# using a fuzzing-capable host or toolchain. +# +# Args: +# - enable_test_if: (optional) Passed as `enable_if` to the unit test. +# Remaining arguments are the same as `pw_executable`. +# template("pw_fuzzer") { - # This currently is ONLY supported on Linux and Mac using clang (debug). - # TODO(pwbug/179): Add Windows here after testing. - fuzzing_platforms = [ - "linux", - "mac", - ] - - fuzzing_toolchains = - [ get_path_info("$dir_pigweed/targets/host:host_clang_fuzz", "abspath") ] - - # This is how GN says 'elem in list': - can_fuzz = fuzzing_platforms + [ host_os ] - [ host_os ] != fuzzing_platforms - - can_fuzz = fuzzing_toolchains + [ current_toolchain ] - - [ current_toolchain ] != fuzzing_toolchains && can_fuzz - - if (can_fuzz && pw_toolchain_SANITIZERS != []) { - # Build the actual fuzzer using the fuzzing config. - pw_executable(target_name) { - forward_variables_from(invoker, "*", [ "visibility" ]) - forward_variables_from(invoker, [ "visibility" ]) - - if (!defined(deps)) { - deps = [] - } - deps += [ dir_pw_fuzzer ] - - if (!defined(configs)) { - configs = [] - } - if (pw_toolchain_OSS_FUZZ_ENABLED) { - configs += [ "$dir_pw_fuzzer:oss_fuzz" ] - } else { - configs += [ "$dir_pw_fuzzer:fuzzing" ] - } - - _fuzzer_output_dir = "${target_out_dir}/bin" - if (defined(invoker.output_dir)) { - _fuzzer_output_dir = invoker.output_dir - } - output_dir = _fuzzer_output_dir - - # Metadata for this fuzzer when used as part of a pw_test_group target. - metadata = { - tests = [ - { - type = "fuzzer" - test_name = target_name - test_directory = rebase_path(output_dir, root_build_dir) - }, - ] - } - } - - # No-op target to satisfy `pw_test_group`. It is empty as we don't want to - # automatically run fuzzers. - group(target_name + ".run") { + if (!pw_toolchain_FUZZING_ENABLED) { + pw_error(target_name) { + message_lines = [ "Toolchain does not enable fuzzing." ] } - - # No-op target to satisfy `pw_test`. It is empty as we don't need a separate - # lib target. - group(target_name + ".lib") { + not_needed(invoker, "*") + } else if (pw_toolchain_SANITIZERS == []) { + pw_error(target_name) { + message_lines = [ "No sanitizer runtime set." ] } + not_needed(invoker, "*") } else { - # Build a unit test that exercise the fuzz target function. - pw_test(target_name) { - # TODO(b/234891784): Re-enable when there's better configurability for - # on-device fuzz testing. - enable_if = false - sources = [] + pw_executable(target_name) { + configs = [] deps = [] - forward_variables_from(invoker, "*", [ "visibility" ]) + forward_variables_from(invoker, + "*", + [ + "enable_test_if", + "visibility", + ]) forward_variables_from(invoker, [ "visibility" ]) - sources += [ "$dir_pw_fuzzer/pw_fuzzer_disabled.cc" ] - deps += [ "$dir_pw_fuzzer:run_as_unit_test" ] + configs += [ "$dir_pw_fuzzer:engine" ] + deps += [ dir_pw_fuzzer ] } } + + pw_test("${target_name}_test") { + deps = [] + forward_variables_from(invoker, "*", [ "visibility" ]) + forward_variables_from(invoker, [ "visibility" ]) + deps += [ "$dir_pw_fuzzer:run_as_unit_test" ] + enable_if = !defined(enable_test_if) || enable_test_if + } } diff --git a/pw_fuzzer/oss_fuzz.gni b/pw_fuzzer/oss_fuzz.gni deleted file mode 100644 index 3a2143822..000000000 --- a/pw_fuzzer/oss_fuzz.gni +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2019 The Pigweed Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -# TODO(aarongreen): Do some minimal parsing on the environment variables to -# identify conflicting configs. -oss_fuzz_extra_cflags_c = string_split(getenv("CFLAGS")) -oss_fuzz_extra_cflags_cc = string_split(getenv("CXXFLAGS")) -oss_fuzz_extra_ldflags = string_split(getenv("LDFLAGS")) - -# TODO(pwbug/184): OSS-Fuzz sets -stdlib=libc++, but pw_minimal_cpp_stdlib -# sets -nostdinc++. Find a more flexible mechanism to achieve this and -# similar needs (like removing -fno-rtti fro UBSan). -oss_fuzz_extra_cflags_cc += [ "-Wno-unused-command-line-argument" ] diff --git a/pw_hdlc/encoder_test.cc b/pw_hdlc/encoder_test.cc index df7a1257f..3f23c3db0 100644 --- a/pw_hdlc/encoder_test.cc +++ b/pw_hdlc/encoder_test.cc @@ -46,11 +46,11 @@ constexpr uint8_t kEncodedAddress = (kAddress << 1) | 1; class WriteUnnumberedFrame : public ::testing::Test { protected: - WriteUnnumberedFrame() : writer_(buffer_) {} + WriteUnnumberedFrame() : buffer_{}, writer_(buffer_) {} - stream::MemoryWriter writer_; // Allocate a buffer that will fit any 7-byte payload. std::array<byte, MaxEncodedFrameSize(7)> buffer_; + stream::MemoryWriter writer_; }; constexpr byte kUnnumberedControl = byte{0x3}; diff --git a/pw_hdlc/rpc_example/example_script.py b/pw_hdlc/rpc_example/example_script.py index ec071e97f..36a3cf85d 100755 --- a/pw_hdlc/rpc_example/example_script.py +++ b/pw_hdlc/rpc_example/example_script.py @@ -18,7 +18,7 @@ import argparse import os from pathlib import Path -import serial # type: ignore +import serial from pw_hdlc.rpc import HdlcRpcClient, default_channels diff --git a/pw_ide/py/cpp_test.py b/pw_ide/py/cpp_test.py index d0d9e708f..87d57f3e0 100644 --- a/pw_ide/py/cpp_test.py +++ b/pw_ide/py/cpp_test.py @@ -808,7 +808,6 @@ class TestCppIdeFeaturesState(PwIdeTestCase): with self.make_temp_file(compdb_symlink_path), self.make_temp_file( cache_symlink_path ): - # Set the second target, which should replace the symlinks CppIdeFeaturesState(settings).current_target = targets[1] @@ -868,7 +867,6 @@ class TestCppIdeFeaturesState(PwIdeTestCase): with self.make_temp_file(compdb_symlink_path), self.make_temp_file( cache_symlink_path ): - # Set the second target, which should replace the symlinks CppIdeFeaturesState(settings).current_target = targets[1] diff --git a/pw_ide/py/pw_ide/editors.py b/pw_ide/py/pw_ide/editors.py index 311f9a9ed..f59e35346 100644 --- a/pw_ide/py/pw_ide/editors.py +++ b/pw_ide/py/pw_ide/editors.py @@ -123,14 +123,14 @@ class Json5FileFormat(_StructuredFileFormat): # Allows constraining to dicts and dict subclasses, while also constraining to # the *same* dict subclass. -_TDictLike = TypeVar('_TDictLike', bound=Dict) +_DictLike = TypeVar('_DictLike', bound=Dict) def dict_deep_merge( - src: _TDictLike, - dest: _TDictLike, - ctor: Optional[Callable[[], _TDictLike]] = None, -) -> _TDictLike: + src: _DictLike, + dest: _DictLike, + ctor: Optional[Callable[[], _DictLike]] = None, +) -> _DictLike: """Deep merge dict-like `src` into dict-like `dest`. `dest` is mutated in place and also returned. @@ -480,20 +480,20 @@ SettingsFilePrefixes = Dict[SettingsLevel, str] # name of that settings file, without the extension. # TODO(chadnorvell): Would be great to constrain this to enums, but bound= # doesn't do what we want with Enum or EnumMeta. -_TSettingsType = TypeVar('_TSettingsType') +_SettingsTypeT = TypeVar('_SettingsTypeT') # Maps each settings type with the callback that generates the default settings # for that settings type. -EditorSettingsTypesWithDefaults = Dict[_TSettingsType, DefaultSettingsCallback] +EditorSettingsTypesWithDefaults = Dict[_SettingsTypeT, DefaultSettingsCallback] -class EditorSettingsManager(Generic[_TSettingsType]): +class EditorSettingsManager(Generic[_SettingsTypeT]): """Manages all settings for a particular editor. This is where you interact with an editor's settings (actually in a subclass of this class, not here). Initializing this class sets up access to one or more settings files for an editor (determined by - ``_TSettingsType``, fulfilled by an enum that defines each of an editor's + ``_SettingsTypeT``, fulfilled by an enum that defines each of an editor's settings files), along with the cascading settings levels. """ @@ -509,7 +509,7 @@ class EditorSettingsManager(Generic[_TSettingsType]): # These must be overridden in child classes. default_settings_dir: Path = None # type: ignore file_format: _StructuredFileFormat = _StructuredFileFormat() - types_with_defaults: EditorSettingsTypesWithDefaults[_TSettingsType] = {} + types_with_defaults: EditorSettingsTypesWithDefaults[_SettingsTypeT] = {} def __init__( self, @@ -517,7 +517,7 @@ class EditorSettingsManager(Generic[_TSettingsType]): settings_dir: Optional[Path] = None, file_format: Optional[_StructuredFileFormat] = None, types_with_defaults: Optional[ - EditorSettingsTypesWithDefaults[_TSettingsType] + EditorSettingsTypesWithDefaults[_SettingsTypeT] ] = None, ): if SettingsLevel.ACTIVE in self.__class__.prefixes: @@ -564,7 +564,7 @@ class EditorSettingsManager(Generic[_TSettingsType]): # each settings type. Those settings definitions may be stored in files # or not. self._settings_definitions: Dict[ - SettingsLevel, Dict[_TSettingsType, EditorSettingsDefinition] + SettingsLevel, Dict[_SettingsTypeT, EditorSettingsDefinition] ] = {} self._settings_types = tuple(self._types_with_defaults.keys()) @@ -597,19 +597,19 @@ class EditorSettingsManager(Generic[_TSettingsType]): self._settings_dir, name, self._file_format ) - def default(self, settings_type: _TSettingsType): + def default(self, settings_type: _SettingsTypeT): """Default settings for the provided settings type.""" return self._settings_definitions[SettingsLevel.DEFAULT][settings_type] - def project(self, settings_type: _TSettingsType): + def project(self, settings_type: _SettingsTypeT): """Project settings for the provided settings type.""" return self._settings_definitions[SettingsLevel.PROJECT][settings_type] - def user(self, settings_type: _TSettingsType): + def user(self, settings_type: _SettingsTypeT): """User settings for the provided settings type.""" return self._settings_definitions[SettingsLevel.USER][settings_type] - def active(self, settings_type: _TSettingsType): + def active(self, settings_type: _SettingsTypeT): """Active settings for the provided settings type.""" return self._settings_definitions[SettingsLevel.ACTIVE][settings_type] diff --git a/pw_ide/py/setup.cfg b/pw_ide/py/setup.cfg index b166ce1ff..c7334c3ab 100644 --- a/pw_ide/py/setup.cfg +++ b/pw_ide/py/setup.cfg @@ -21,7 +21,7 @@ description = Tools for Pigweed editor and IDE support [options] packages = find: install_requires = - json5 ==0.9.10 + json5>=0.9.10 [options.entry_points] console_scripts = pw-ide = pw_ide.__main__:main diff --git a/pw_log/protobuf.rst b/pw_log/protobuf.rst index c65171f8f..ff7335f8c 100644 --- a/pw_log/protobuf.rst +++ b/pw_log/protobuf.rst @@ -89,11 +89,14 @@ Encoding logs to the ``log.proto`` format can be performed using the helpers provided in the ``pw_log/proto_utils.h`` header. Separate helpers are provided for encoding tokenized logs and string-based logs. +The following example shows a :c:func:`pw_log_tokenized_HandleLog` +implementation that encodes the results to a protobuf. + .. code-block:: cpp #include "pw_log/proto_utils.h" - extern "C" void pw_log_tokenized_HandleLog(( + extern "C" void pw_log_tokenized_HandleLog( uint32_t payload, const uint8_t data[], size_t size) { pw::log_tokenized::Metadata metadata(payload); std::byte log_buffer[kLogBufferSize]; diff --git a/pw_log_rpc/docs.rst b/pw_log_rpc/docs.rst index 4a9db1338..0b30356f0 100644 --- a/pw_log_rpc/docs.rst +++ b/pw_log_rpc/docs.rst @@ -28,9 +28,8 @@ Set up the :ref:`module-pw_log_tokenized` log backend. 3. Connect the tokenized logging handler to the MultiSink --------------------------------------------------------- Create a :ref:`MultiSink <module-pw_multisink>` instance to buffer log entries. -Then, make the log backend handler, :cpp:func:`pw_log_tokenized_HandleLog`, -encode log entries in the ``log::LogEntry`` format, and add them to the -``MultiSink``. +Then, make the log backend handler, :c:func:`pw_log_tokenized_HandleLog`, encode +log entries in the ``log::LogEntry`` format, and add them to the ``MultiSink``. 4. Create log drains and filters -------------------------------- diff --git a/pw_log_rpc/log_filter_service_test.cc b/pw_log_rpc/log_filter_service_test.cc index fe8db8f5f..942d371e2 100644 --- a/pw_log_rpc/log_filter_service_test.cc +++ b/pw_log_rpc/log_filter_service_test.cc @@ -41,7 +41,6 @@ class FilterServiceTest : public ::testing::Test { FilterServiceTest() : filter_map_(filters_) {} protected: - FilterMap filter_map_; static constexpr size_t kMaxFilterRules = 4; std::array<Filter::Rule, kMaxFilterRules> rules1_; std::array<Filter::Rule, kMaxFilterRules> rules2_; @@ -58,6 +57,7 @@ class FilterServiceTest : public ::testing::Test { Filter(filter_id2_, rules2_), Filter(filter_id3_, rules3_), }; + FilterMap filter_map_; }; TEST_F(FilterServiceTest, GetFilterIds) { diff --git a/pw_log_tokenized/BUILD.gn b/pw_log_tokenized/BUILD.gn index cc1b74262..3e2b16181 100644 --- a/pw_log_tokenized/BUILD.gn +++ b/pw_log_tokenized/BUILD.gn @@ -42,20 +42,12 @@ config("backend_config") { # This target provides the backend for pw_log. pw_source_set("pw_log_tokenized") { - public_configs = [ - ":backend_config", - ":public_include_path", - ] + public_configs = [ ":backend_config" ] public_deps = [ - ":config", ":handler.facade", # Depend on the facade to avoid circular dependencies. - ":metadata", - dir_pw_tokenizer, - ] - public = [ - "public/pw_log_tokenized/log_tokenized.h", - "public_overrides/pw_log_backend/log_backend.h", + ":headers", ] + public = [ "public_overrides/pw_log_backend/log_backend.h" ] sources = [ "log_tokenized.cc" ] } @@ -65,6 +57,22 @@ config("backwards_compatibility_config") { visibility = [ ":*" ] } +pw_source_set("headers") { + visibility = [ ":*" ] + public_configs = [ ":public_include_path" ] + public_deps = [ + ":config", + ":metadata", + + # TODO(hepler): Remove this dependency when all projects have migrated to + # the new pw_log_tokenized handler. + "$dir_pw_tokenizer:global_handler_with_payload", + dir_pw_preprocessor, + dir_pw_tokenizer, + ] + public = [ "public/pw_log_tokenized/log_tokenized.h" ] +} + # The old pw_tokenizer_GLOBAL_HANDLER_WITH_PAYLOAD_BACKEND backend may still be # in use by projects that have not switched to the new pw_log_tokenized facade. # Use the old backend as a stand-in for the new backend if it is set. @@ -156,16 +164,10 @@ pw_source_set("base64_over_hdlc") { } pw_test_group("tests") { - tests = [ ":metadata_test" ] - - # TODO(b/269354373): The Windows MinGW compiler fails to link - # log_tokenized_test.cc because log_tokenized.cc refers to the log handler, - # which is not defined, even though _pw_log_tokenized_EncodeTokenizedLog is - # never called. Remove this check when the Windows build is consistent with - # other builds. - if (current_os != "win" || _new_backend_is_set || _old_backend_is_set) { - tests += [ ":log_tokenized_test" ] - } + tests = [ + ":log_tokenized_test", + ":metadata_test", + ] } pw_test("log_tokenized_test") { @@ -175,7 +177,7 @@ pw_test("log_tokenized_test") { "pw_log_tokenized_private/test_utils.h", ] deps = [ - ":pw_log_tokenized", + ":headers", dir_pw_preprocessor, ] } diff --git a/pw_log_tokenized/docs.rst b/pw_log_tokenized/docs.rst index 662c1dfc3..24a6d8d4a 100644 --- a/pw_log_tokenized/docs.rst +++ b/pw_log_tokenized/docs.rst @@ -11,15 +11,10 @@ C++ backend ``pw_log_tokenized`` provides a backend for ``pw_log`` that tokenizes log messages with the ``pw_tokenizer`` module. The log level, 16-bit tokenized module name, and flags bits are passed through the payload argument. The macro -eventually passes logs to the ``pw_log_tokenized_HandleLog`` function, which -must be implemented by the application. +eventually passes logs to the :c:func:`pw_log_tokenized_HandleLog` function, +which must be implemented by the application. -.. c:function: void pw_log_tokenized_HandleLog(uint32_t metadata, const uint8_t[] message, size_t size_bytes) - - Function that is called for each log message. The metadata uint32_t can be - converted to a :cpp:class`pw::log::tokenized::Metadata`. The message is passed - as a pointer to a buffer and a size. The pointer is invalidated after this - function returns, so the buffer must be copied. +.. doxygenfunction:: pw_log_tokenized_HandleLog Example implementation: @@ -135,20 +130,18 @@ bits allocated is excluded from the log metadata. Creating and reading Metadata payloads ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -A C++ class is provided to facilitate the creation and interpretation of packed -log metadata payloads. The ``GenericMetadata`` class allows flags, log level, -line number, and a module identifier to be packed into bit fields of -configurable size. The ``Metadata`` class simplifies the bit field width -templatization of ``GenericMetadata`` by pulling from this module's -configuration options. In most cases, it's recommended to use ``Metadata`` to -create or read metadata payloads. +``pw_log_tokenized`` provides a C++ class to facilitate the creation and +interpretation of packed log metadata payloads. + +.. doxygenclass:: pw::log_tokenized::GenericMetadata +.. doxygentypedef:: pw::log_tokenized::Metadata -A ``Metadata`` object can be created from a ``uint32_t`` allowing -various peices of metadata to be read from the payload as seen below: +The following example shows that a ``Metadata`` object can be created from a +``uint32_t`` log metadata payload. .. code-block:: cpp - extern "C" void pw_log_tokenized_HandleLog(( + extern "C" void pw_log_tokenized_HandleLog( uint32_t payload, const uint8_t message[], size_t size_bytes) { diff --git a/pw_log_tokenized/metadata_test.cc b/pw_log_tokenized/metadata_test.cc index 74de0a519..4d84c5aaf 100644 --- a/pw_log_tokenized/metadata_test.cc +++ b/pw_log_tokenized/metadata_test.cc @@ -20,7 +20,7 @@ namespace pw::log_tokenized { namespace { TEST(Metadata, NoLineBits) { - using NoLineBits = internal::GenericMetadata<6, 0, 10, 16>; + using NoLineBits = GenericMetadata<6, 0, 10, 16>; constexpr NoLineBits test1 = NoLineBits::Set<0, 0, 0>(); static_assert(test1.level() == 0); @@ -42,7 +42,7 @@ TEST(Metadata, NoLineBits) { } TEST(Metadata, NoFlagBits) { - using NoFlagBits = internal::GenericMetadata<3, 13, 0, 16>; + using NoFlagBits = GenericMetadata<3, 13, 0, 16>; constexpr NoFlagBits test1 = NoFlagBits::Set<0, 0, 0, 0>(); static_assert(test1.level() == 0); diff --git a/pw_log_tokenized/public/pw_log_tokenized/handler.h b/pw_log_tokenized/public/pw_log_tokenized/handler.h index f15ee5bac..40ac0f7d8 100644 --- a/pw_log_tokenized/public/pw_log_tokenized/handler.h +++ b/pw_log_tokenized/public/pw_log_tokenized/handler.h @@ -20,6 +20,10 @@ PW_EXTERN_C_START +/// Function that is called for each log message. The metadata `uint32_t` can be +/// converted to a @cpp_type{pw::log_tokenized::Metadata}. The message is passed +/// as a pointer to a buffer and a size. The pointer is invalidated after this +/// function returns, so the buffer must be copied. void pw_log_tokenized_HandleLog(uint32_t metadata, const uint8_t encoded_message[], size_t size_bytes); diff --git a/pw_log_tokenized/public/pw_log_tokenized/metadata.h b/pw_log_tokenized/public/pw_log_tokenized/metadata.h index 7a87bd45e..5d78e3b38 100644 --- a/pw_log_tokenized/public/pw_log_tokenized/metadata.h +++ b/pw_log_tokenized/public/pw_log_tokenized/metadata.h @@ -41,8 +41,17 @@ class BitField<T, 0, kShift> { static constexpr T Shift(T) { return 0; } }; +} // namespace internal + // This class, which is aliased to pw::log_tokenized::Metadata below, is used to // access the log metadata packed into the tokenizer's payload argument. +// +/// `GenericMetadata` facilitates the creation and interpretation of packed +/// log metadata payloads. The `GenericMetadata` class allows flags, log level, +/// line number, and a module identifier to be packed into bit fields of +/// configurable size. +/// +/// Typically, the `Metadata` alias should be used instead. template <unsigned kLevelBits, unsigned kLineBits, unsigned kFlagBits, @@ -60,35 +69,37 @@ class GenericMetadata { return GenericMetadata(BitsFromMetadata(log_level, module, flags, line)); } - // Only use this constructor for creating metadata from runtime values. This - // constructor is unable to warn at compilation when values will not fit in - // the specified bit field widths. + /// Only use this constructor for creating metadata from runtime values. This + /// constructor is unable to warn at compilation when values will not fit in + /// the specified bit field widths. constexpr GenericMetadata(T log_level, T module, T flags, T line) : value_(BitsFromMetadata(log_level, module, flags, line)) {} constexpr GenericMetadata(T value) : value_(value) {} - // The log level of this message. + /// The log level of this message. constexpr T level() const { return Level::Get(value_); } - // The line number of the log call. The first line in a file is 1. If the line - // number is 0, it was too large to be stored. + /// The line number of the log call. The first line in a file is 1. If the + /// line number is 0, it was too large to be stored. constexpr T line_number() const { return Line::Get(value_); } - // The flags provided to the log call. + /// The flags provided to the log call. constexpr T flags() const { return Flags::Get(value_); } - // The 16 bit tokenized version of the module name (PW_LOG_MODULE_NAME). + /// The 16-bit tokenized version of the module name + /// (@c_macro{PW_LOG_MODULE_NAME}). constexpr T module() const { return Module::Get(value_); } - // The underlying packed metadata. + /// The underlying packed metadata. constexpr T value() const { return value_; } private: - using Level = BitField<T, kLevelBits, 0>; - using Line = BitField<T, kLineBits, kLevelBits>; - using Flags = BitField<T, kFlagBits, kLevelBits + kLineBits>; - using Module = BitField<T, kModuleBits, kLevelBits + kLineBits + kFlagBits>; + using Level = internal::BitField<T, kLevelBits, 0>; + using Line = internal::BitField<T, kLineBits, kLevelBits>; + using Flags = internal::BitField<T, kFlagBits, kLevelBits + kLineBits>; + using Module = + internal::BitField<T, kModuleBits, kLevelBits + kLineBits + kFlagBits>; static constexpr T BitsFromMetadata(T log_level, T module, T flags, T line) { return Level::Shift(log_level) | Module::Shift(module) | @@ -101,12 +112,16 @@ class GenericMetadata { sizeof(value_) * 8); }; -} // namespace internal - -using Metadata = internal::GenericMetadata<PW_LOG_TOKENIZED_LEVEL_BITS, - PW_LOG_TOKENIZED_LINE_BITS, - PW_LOG_TOKENIZED_FLAG_BITS, - PW_LOG_TOKENIZED_MODULE_BITS>; +/// The `Metadata` alias simplifies the bit field width templatization of +/// `GenericMetadata` by pulling from this module's configuration options. In +/// most cases, it's recommended to use `Metadata` to create or read metadata +/// payloads. +/// +/// A `Metadata` object can be created from a `uint32_t`. +using Metadata = GenericMetadata<PW_LOG_TOKENIZED_LEVEL_BITS, + PW_LOG_TOKENIZED_LINE_BITS, + PW_LOG_TOKENIZED_FLAG_BITS, + PW_LOG_TOKENIZED_MODULE_BITS>; } // namespace log_tokenized } // namespace pw diff --git a/pw_log_zephyr/CMakeLists.txt b/pw_log_zephyr/CMakeLists.txt index 08946fa6d..3f39df97d 100644 --- a/pw_log_zephyr/CMakeLists.txt +++ b/pw_log_zephyr/CMakeLists.txt @@ -14,24 +14,34 @@ include($ENV{PW_ROOT}/pw_build/pigweed.cmake) -if(NOT CONFIG_PIGWEED_LOG) - return() +if(CONFIG_PIGWEED_LOG_ZEPHYR) + pw_add_library(pw_log_zephyr STATIC + HEADERS + public/pw_log_zephyr/log_zephyr.h + public/pw_log_zephyr/config.h + public_overrides/pw_log_backend/log_backend.h + PUBLIC_INCLUDES + public + public_overrides + PUBLIC_DEPS + pw_log.facade + zephyr_interface + SOURCES + log_zephyr.cc + PRIVATE_DEPS + pw_preprocessor + ) + zephyr_link_libraries(pw_log_zephyr) endif() -pw_add_library(pw_log_zephyr STATIC - HEADERS - public/pw_log_zephyr/log_zephyr.h - public/pw_log_zephyr/config.h - public_overrides/pw_log_backend/log_backend.h - PUBLIC_INCLUDES - public - public_overrides - PUBLIC_DEPS - pw_log.facade - zephyr_interface - SOURCES - log_zephyr.cc - PRIVATE_DEPS - pw_preprocessor -) -zephyr_link_libraries(pw_log_zephyr) +if(CONFIG_PIGWEED_LOG_TOKENIZED) + pw_add_library(pw_log_zephyr.tokenized_handler STATIC + SOURCES + pw_log_zephyr_tokenized_handler.cc + PRIVATE_DEPS + pw_log_tokenized.handler + pw_tokenizer + ) + zephyr_link_libraries(pw_log pw_log_zephyr.tokenized_handler) + zephyr_include_directories(public_overrides) +endif() diff --git a/pw_log_zephyr/Kconfig b/pw_log_zephyr/Kconfig index 472b54c64..59a971be9 100644 --- a/pw_log_zephyr/Kconfig +++ b/pw_log_zephyr/Kconfig @@ -12,17 +12,41 @@ # License for the specific language governing permissions and limitations under # the License. -menuconfig PIGWEED_LOG - bool "Enable Pigweed logging library (pw_log)" +choice PIGWEED_LOG + prompt "Logging backend used" + help + The type of Zephyr pw_log backend to use. + +config PIGWEED_LOG_ZEPHYR + bool "Zephyr logging for PW_LOG_* statements" select PIGWEED_PREPROCESSOR help Once the Pigweed logging is enabled, all Pigweed logs via PW_LOG_*() will - go to the "pigweed" Zephyr logging module. + be routed to the Zephyr logging system. This means that: + - PW_LOG_LEVEL_DEBUG maps to Zephyr's LOG_LEVEL_DBG + - PW_LOG_LEVEL_INFO maps to Zephyr's LOG_LEVEL_INF + - PW_LOG_LEVEL_WARN maps to Zephyr's LOG_LEVEL_WRN + - PW_LOG_LEVEL_ERROR maps to Zephyr's LOG_LEVEL_ERR + - PW_LOG_LEVEL_CRITICAL maps to Zephyr's LOG_LEVEL_ERR + - PW_LOG_LEVEL_FATAL maps to Zephyr's LOG_LEVEL_ERR + +config PIGWEED_LOG_TOKENIZED + bool "Maps all Zephyr log macros to tokenized PW_LOG_* macros" + select PIGWEED_PREPROCESSOR + select PIGWEED_TOKENIZER + select LOG_CUSTOM_HEADER + help + Map all the Zephyr log macros to use Pigweed's then use the + 'pw_log_tokenized' target as the logging backend in order to + automatically tokenize all the logging strings. This means that Pigweed + will also tokenize all of Zephyr's logging statements. + +endchoice -if PIGWEED_LOG +if PIGWEED_LOG_ZEPHYR || PIGWEED_LOG_TOKENIZED module = PIGWEED module-str = "pigweed" source "subsys/logging/Kconfig.template.log_config" -endif # PIGWEED_LOG +endif # PIGWEED_LOG_ZEPHYR || PIGWEED_LOG_TOKENIZED diff --git a/pw_log_zephyr/docs.rst b/pw_log_zephyr/docs.rst index 84b8db9ac..beb6a691f 100644 --- a/pw_log_zephyr/docs.rst +++ b/pw_log_zephyr/docs.rst @@ -1,17 +1,31 @@ .. _module-pw_log_zephyr: -================ +============= pw_log_zephyr -================ +============= -------- Overview -------- -This interrupt backend implements the ``pw_log`` facade. To enable, set -``CONFIG_PIGWEED_LOG=y``. After that, logging can be controlled via the standard -`Kconfig options <https://docs.zephyrproject.org/latest/reference/logging/index.html#global-kconfig-options>`_. -All logs made through `PW_LOG_*` are logged to the Zephyr logging module -``pigweed``. +This interrupt backend implements the ``pw_log`` facade. Currently, two +separate Pigweed backends are implemented. One that uses the plain Zephyr +logging framework and routes Pigweed's logs to Zephyr. While another maps +the Zephyr logging macros to Pigweed's tokenized logging. + +Using Zephyr logging +-------------------- +To enable, set ``CONFIG_PIGWEED_LOG_ZEPHYR=y``. After that, logging can be +controlled via the standard `Kconfig options`_. All logs made through +`PW_LOG_*` are logged to the Zephyr logging module ``pigweed``. In this +model, the Zephyr logging is set as ``pw_log``'s backend. + +Using Pigweed tokenized logging +------------------------------- +Using the pigweed logging can be done by enabling +``CONFIG_PIGWEED_LOG_TOKENIZED=y``. At that point ``pw_log_tokenized`` is set +as the backedn for ``pw_log`` and all Zephyr logs are routed to Pigweed's +logging facade. This means that any logging statements made in Zephyr itself +are also tokenized. Setting the log level --------------------- @@ -37,3 +51,5 @@ levels to their closest Zephyr counterparts: Alternatively, it is also possible to set the Zephyr logging level directly via ``CONFIG_PIGWEED_LOG_LEVEL``. + +.. _`Kconfig options`: https://docs.zephyrproject.org/latest/reference/logging/index.html#global-kconfig-options diff --git a/pw_log_zephyr/public_overrides/zephyr_custom_log.h b/pw_log_zephyr/public_overrides/zephyr_custom_log.h new file mode 100644 index 000000000..8485d5e02 --- /dev/null +++ b/pw_log_zephyr/public_overrides/zephyr_custom_log.h @@ -0,0 +1,35 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#pragma once + +#include <zephyr/sys/__assert.h> + +// If static_assert wasn't defined by zephyr/sys/__assert.h that means it's not +// supported, just ignore it. +#ifndef static_assert +#define static_assert(...) +#endif + +#include <pw_log/log.h> + +#undef LOG_DBG +#undef LOG_INF +#undef LOG_WRN +#undef LOG_ERR + +#define LOG_DBG(format, ...) PW_LOG_DEBUG(format, ##__VA_ARGS__) +#define LOG_INF(format, ...) PW_LOG_INFO(format, ##__VA_ARGS__) +#define LOG_WRN(format, ...) PW_LOG_WARN(format, ##__VA_ARGS__) +#define LOG_ERR(format, ...) PW_LOG_ERROR(format, ##__VA_ARGS__) diff --git a/pw_log_zephyr/pw_log_zephyr_tokenized_handler.cc b/pw_log_zephyr/pw_log_zephyr_tokenized_handler.cc new file mode 100644 index 000000000..bdc64ed49 --- /dev/null +++ b/pw_log_zephyr/pw_log_zephyr_tokenized_handler.cc @@ -0,0 +1,31 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#include <zephyr/logging/log_backend.h> +#include <zephyr/logging/log_msg.h> + +#include "pw_log_tokenized/handler.h" + +namespace pw::log_tokenized { + +extern "C" void pw_log_tokenized_HandleLog(uint32_t metadata, + const uint8_t log_buffer[], + size_t size_bytes) { + ARG_UNUSED(metadata); + ARG_UNUSED(log_buffer); + ARG_UNUSED(size_bytes); + // TODO(asemjonovs): implement this function +} + +} // namespace pw::log_tokenized diff --git a/pw_module/py/pw_module/create.py b/pw_module/py/pw_module/create.py index b5ff6d398..d53dca03a 100644 --- a/pw_module/py/pw_module/create.py +++ b/pw_module/py/pw_module/create.py @@ -899,7 +899,7 @@ def _create_module( root=project_root, modules_list=modules_file, modules_gni_file=modules_gni_file, - warn_only=None, + mode=generate_modules_lists.Mode.UPDATE, ) print(' modify ' + str(modules_gni_file.relative_to(Path.cwd()))) diff --git a/pw_multisink/multisink_test.cc b/pw_multisink/multisink_test.cc index b466ba09a..3489a0629 100644 --- a/pw_multisink/multisink_test.cc +++ b/pw_multisink/multisink_test.cc @@ -52,7 +52,7 @@ class MultiSinkTest : public ::testing::Test { static constexpr size_t kEntryBufferSize = 1024; static constexpr size_t kBufferSize = 5 * kEntryBufferSize; - MultiSinkTest() : multisink_(buffer_) {} + MultiSinkTest() : buffer_{}, multisink_(buffer_) {} // Expects the peeked or popped message to equal the provided non-empty // message, and the drop count to match. If `expected_message` is empty, the diff --git a/pw_perf_test/public/pw_perf_test/perf_test.h b/pw_perf_test/public/pw_perf_test/perf_test.h index 64f6e6411..e092087f9 100644 --- a/pw_perf_test/public/pw_perf_test/perf_test.h +++ b/pw_perf_test/public/pw_perf_test/perf_test.h @@ -23,7 +23,7 @@ #include "pw_preprocessor/arguments.h" #define PW_PERF_TEST(name, function, ...) \ - ::pw::perf_test::internal::TestInfo PwPerfTest_##name( \ + const ::pw::perf_test::internal::TestInfo PwPerfTest_##name( \ #name, [](::pw::perf_test::State& pw_perf_test_state) { \ static_cast<void>( \ function(pw_perf_test_state PW_COMMA_ARGS(__VA_ARGS__))); \ @@ -59,6 +59,7 @@ class Framework { : event_handler_(nullptr), tests_(nullptr), run_info_{.total_tests = 0, .default_iterations = kDefaultIterations} {} + static Framework& Get() { return framework_; } void RegisterEventHandler(EventHandler& event_handler) { @@ -84,7 +85,7 @@ class Framework { class TestInfo { public: - constexpr TestInfo(const char* test_name, void (*function_body)(State&)) + TestInfo(const char* test_name, void (*function_body)(State&)) : run_(function_body), test_name_(test_name) { // Once a TestInfo object is created by the macro, this adds itself to the // list of registered tests diff --git a/pw_presubmit/docs.rst b/pw_presubmit/docs.rst index 3e48ad15e..5be527bc9 100644 --- a/pw_presubmit/docs.rst +++ b/pw_presubmit/docs.rst @@ -192,6 +192,34 @@ all use language-specific formatters like clang-format or black. These will suggest fixes using ``pw format --fix``. +Options for code formatting can be specified in the ``pigweed.json`` file +(see also :ref:`SEED-0101 <seed-0101>`). These apply to both ``pw presubmit`` +steps that check code formatting and ``pw format`` commands that either check +or fix code formatting. + +* ``python_formatter``: Choice of Python formatter. Options are ``black`` (used + by Pigweed itself) and ``yapf`` (the default). +* ``black_path``: If ``python_formatter`` is ``black``, use this as the + executable instead of ``black``. + +.. TODO(b/264578594) Add exclude to pigweed.json file. +.. * ``exclude``: List of path regular expressions to ignore. + +Example section from a ``pigweed.json`` file: + +.. code-block:: + + { + "pw": { + "pw_presubmit": { + "format": { + "python_formatter": "black", + "black_path": "black" + } + } + } + } + Sorted Blocks ^^^^^^^^^^^^^ Blocks of code can be required to be kept in sorted order using comments like diff --git a/pw_presubmit/py/pw_presubmit/build.py b/pw_presubmit/py/pw_presubmit/build.py index 167de408b..6ef59e880 100644 --- a/pw_presubmit/py/pw_presubmit/build.py +++ b/pw_presubmit/py/pw_presubmit/build.py @@ -520,7 +520,7 @@ def test_server(executable: str, output_dir: Path): yield finally: - proc.terminate() + proc.terminate() # pylint: disable=used-before-assignment @filter_paths( diff --git a/pw_presubmit/py/pw_presubmit/format_code.py b/pw_presubmit/py/pw_presubmit/format_code.py index 03f1f3712..9b073ea05 100755 --- a/pw_presubmit/py/pw_presubmit/format_code.py +++ b/pw_presubmit/py/pw_presubmit/format_code.py @@ -58,6 +58,7 @@ from pw_presubmit.presubmit import FileFilter from pw_presubmit import ( cli, FormatContext, + FormatOptions, git_repo, owners_checks, PresubmitContext, @@ -266,9 +267,6 @@ def fix_py_format_yapf(ctx: _Context) -> Dict[Path, str]: return {} -BLACK = 'black' - - def _enumerate_black_configs() -> Iterable[Path]: if directory := os.environ.get('PW_PROJECT_ROOT'): yield Path(directory, '.black.toml') @@ -293,10 +291,11 @@ def _black_config_args() -> Sequence[Union[str, Path]]: def _black_multiple_files(ctx: _Context) -> Tuple[str, ...]: + black = ctx.format_options.black_path changed_paths: List[str] = [] for line in ( log_run( - [BLACK, '--check', *_black_config_args(), *ctx.paths], + [black, '--check', *_black_config_args(), *ctx.paths], capture_output=True, ) .stderr.decode() @@ -324,7 +323,7 @@ def check_py_format_black(ctx: _Context) -> Dict[Path, str]: build.write_bytes(data) proc = log_run( - [BLACK, *_black_config_args(), build], + [ctx.format_options.black_path, *_black_config_args(), build], capture_output=True, ) if proc.returncode: @@ -354,7 +353,7 @@ def fix_py_format_black(ctx: _Context) -> Dict[Path, str]: continue proc = log_run( - [BLACK, *_black_config_args(), path], + [ctx.format_options.black_path, *_black_config_args(), path], capture_output=True, ) if proc.returncode: @@ -362,6 +361,22 @@ def fix_py_format_black(ctx: _Context) -> Dict[Path, str]: return errors +def check_py_format(ctx: _Context) -> Dict[Path, str]: + if ctx.format_options.python_formatter == 'black': + return check_py_format_black(ctx) + if ctx.format_options.python_formatter == 'yapf': + return check_py_format_yapf(ctx) + raise ValueError(ctx.format_options.python_formatter) + + +def fix_py_format(ctx: _Context) -> Dict[Path, str]: + if ctx.format_options.python_formatter == 'black': + return fix_py_format_black(ctx) + if ctx.format_options.python_formatter == 'yapf': + return fix_py_format_yapf(ctx) + raise ValueError(ctx.format_options.python_formatter) + + _TRAILING_SPACE = re.compile(rb'[ \t]+$', flags=re.MULTILINE) @@ -484,19 +499,11 @@ GO_FORMAT: CodeFormat = CodeFormat( 'Go', FileFilter(endswith=('.go',)), check_go_format, fix_go_format ) -# TODO(b/259595799) Remove yapf support. -PYTHON_FORMAT_YAPF: CodeFormat = CodeFormat( - 'Python', - FileFilter(endswith=('.py',)), - check_py_format_yapf, - fix_py_format_yapf, -) - -PYTHON_FORMAT_BLACK: CodeFormat = CodeFormat( +PYTHON_FORMAT: CodeFormat = CodeFormat( 'Python', FileFilter(endswith=('.py',)), - check_py_format_black, - fix_py_format_black, + check_py_format, + fix_py_format, ) GN_FORMAT: CodeFormat = CodeFormat( @@ -546,7 +553,7 @@ OWNERS_CODE_FORMAT = CodeFormat( fix=fix_owners_format, ) -_CODE_FORMATS_WITHOUT_PYTHON: Tuple[CodeFormat, ...] = ( +CODE_FORMATS: Tuple[CodeFormat, ...] = ( # keep-sorted: start BAZEL_FORMAT, CMAKE_FORMAT, @@ -559,23 +566,14 @@ _CODE_FORMATS_WITHOUT_PYTHON: Tuple[CodeFormat, ...] = ( MARKDOWN_FORMAT, OWNERS_CODE_FORMAT, PROTO_FORMAT, + PYTHON_FORMAT, RST_FORMAT, # keep-sorted: end ) -# TODO(b/259595799) Remove yapf support. -CODE_FORMATS_WITH_YAPF: Tuple[CodeFormat, ...] = ( - *_CODE_FORMATS_WITHOUT_PYTHON, - PYTHON_FORMAT_YAPF, -) - -CODE_FORMATS_WITH_BLACK: Tuple[CodeFormat, ...] = ( - *_CODE_FORMATS_WITHOUT_PYTHON, - PYTHON_FORMAT_BLACK, -) - -# TODO(b/259595799) For downstream compatibility only. -CODE_FORMATS = CODE_FORMATS_WITH_YAPF +# TODO(b/264578594) Remove these lines when these globals aren't referenced. +CODE_FORMATS_WITH_BLACK: Tuple[CodeFormat, ...] = CODE_FORMATS +CODE_FORMATS_WITH_YAPF: Tuple[CodeFormat, ...] = CODE_FORMATS def presubmit_check( @@ -624,7 +622,7 @@ def presubmit_check( def presubmit_checks( *, exclude: Collection[Union[str, Pattern[str]]] = (), - code_formats: Collection[CodeFormat] = CODE_FORMATS_WITH_YAPF, + code_formats: Collection[CodeFormat] = CODE_FORMATS, ) -> Tuple[Callable, ...]: """Returns a tuple with all supported code format presubmit checks. @@ -673,6 +671,7 @@ class CodeFormatter: output_dir=outdir, paths=tuple(self._formats[code_format]), package_root=self.package_root, + format_options=FormatOptions.load(), ) def check(self) -> Dict[Path, str]: @@ -718,7 +717,7 @@ def format_paths_in_repo( exclude: Collection[Pattern[str]], fix: bool, base: str, - code_formats: Collection[CodeFormat] = CODE_FORMATS_WITH_YAPF, + code_formats: Collection[CodeFormat] = CODE_FORMATS, output_directory: Optional[Path] = None, package_root: Optional[Path] = None, ) -> int: @@ -833,10 +832,7 @@ def format_files( return 0 -def arguments( - git_paths: bool, - use_black: bool = False, -) -> argparse.ArgumentParser: +def arguments(git_paths: bool) -> argparse.ArgumentParser: """Creates an argument parser for format_files or format_paths_in_repo.""" parser = argparse.ArgumentParser(description=__doc__) @@ -866,18 +862,6 @@ def arguments( '--fix', action='store_true', help='Apply formatting fixes in place.' ) - # TODO(b/259595799, b/261025545) Remove --code-formats option when - # downstream projects have switched away from yapf. - default_code_formats = CODE_FORMATS_WITH_YAPF - if use_black: - default_code_formats = CODE_FORMATS_WITH_BLACK - parser.add_argument( - '--code-formats', - choices=(CODE_FORMATS_WITH_YAPF, CODE_FORMATS_WITH_BLACK), - default=default_code_formats, - help=argparse.SUPPRESS, - ) - parser.add_argument( '--output-directory', type=Path, @@ -903,7 +887,7 @@ def _pigweed_upstream_main() -> int: Excludes third party sources. """ - args = arguments(git_paths=True, use_black=True).parse_args() + args = arguments(git_paths=True).parse_args() # Exclude paths with third party code from formatting. args.exclude.append(re.compile('^third_party/fuchsia/repo/')) diff --git a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py index 1b4f67122..1369fbc5e 100755 --- a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py +++ b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py @@ -145,6 +145,7 @@ def gn_full_qemu_check(ctx: PresubmitContext): def _gn_combined_build_check_targets() -> Sequence[str]: build_targets = [ + 'check_modules', *_at_all_optimization_levels('stm32f429i'), *_at_all_optimization_levels(f'host_{_HOST_COMPILER}'), 'python.tests', @@ -491,6 +492,27 @@ def bazel_build(ctx: PresubmitContext) -> None: *targets, ) + # Provide some coverage of the FreeRTOS build. + # + # This is just a minimal presubmit intended to ensure we don't break what + # support we have. + # + # TODO(b/271465588): Eventually just build the entire repo for this + # platform. + build.bazel( + ctx, + 'build', + # Designated initializers produce a warning-treated-as-error when + # compiled with -std=c++17. + # + # TODO(b/271299438): Remove this. + '--copt=-Wno-pedantic', + '--platforms=//pw_build/platforms:testonly_freertos', + '//pw_sync/...', + '//pw_thread/...', + '//pw_thread_freertos/...', + ) + def pw_transfer_integration_test(ctx: PresubmitContext) -> None: """Runs the pw_transfer cross-language integration test only. @@ -884,7 +906,6 @@ _EXCLUDE_FROM_TODO_CHECK = ( r'\bpw_doctor/py/pw_doctor/doctor.py', r'\bpw_env_setup/util.sh', r'\bpw_fuzzer/fuzzer.gni', - r'\bpw_fuzzer/oss_fuzz.gni', r'\bpw_i2c/BUILD.gn', r'\bpw_i2c/public/pw_i2c/register_device.h', r'\bpw_kvs/flash_memory.cc', @@ -980,9 +1001,7 @@ PATH_EXCLUSIONS = (re.compile(r'\bthird_party/fuchsia/repo/'),) _LINTFORMAT = ( commit_message_format, copyright_notice, - format_code.presubmit_checks( - code_formats=format_code.CODE_FORMATS_WITH_BLACK - ), + format_code.presubmit_checks(), inclusive_language.presubmit_check.with_filter( exclude=( r'\byarn.lock$', @@ -992,7 +1011,6 @@ _LINTFORMAT = ( cpp_checks.pragma_once, build.bazel_lint, owners_lint_checks, - source_in_build.bazel(SOURCE_FILES_FILTER), source_in_build.gn(SOURCE_FILES_FILTER), source_is_in_cmake_build_warn_only, shell_checks.shellcheck if shutil.which('shellcheck') else (), @@ -1002,6 +1020,11 @@ _LINTFORMAT = ( LINTFORMAT = ( _LINTFORMAT, + # This check is excluded from _LINTFORMAT because it's not quick: it issues + # a bazel query that pulls in all of Pigweed's external dependencies + # (https://stackoverflow.com/q/71024130/1224002). These are cached, but + # after a roll it can be quite slow. + source_in_build.bazel(SOURCE_FILES_FILTER), pw_presubmit.python_checks.check_python_versions, pw_presubmit.python_checks.gn_python_lint, ) diff --git a/pw_presubmit/py/pw_presubmit/presubmit.py b/pw_presubmit/py/pw_presubmit/presubmit.py index bf0802087..ef10a3abe 100644 --- a/pw_presubmit/py/pw_presubmit/presubmit.py +++ b/pw_presubmit/py/pw_presubmit/presubmit.py @@ -77,6 +77,7 @@ import urllib import pw_cli.color import pw_cli.env +import pw_env_setup.config_file from pw_package import package_manager from pw_presubmit import git_repo, tools from pw_presubmit.tools import plural @@ -138,7 +139,6 @@ class PresubmitFailure(Exception): class PresubmitResult(enum.Enum): - PASS = 'PASSED' # Check completed successfully. FAIL = 'FAILED' # Check failed. CANCEL = 'CANCEL' # Check didn't complete. @@ -214,6 +214,25 @@ class Programs(collections.abc.Mapping): return len(self._programs) +@dataclasses.dataclass(frozen=True) +class FormatOptions: + python_formatter: Optional[str] = 'yapf' + black_path: Optional[str] = 'black' + + # TODO(b/264578594) Add exclude to pigweed.json file. + # exclude: Sequence[re.Pattern] = dataclasses.field(default_factory=list) + + @staticmethod + def load() -> 'FormatOptions': + config = pw_env_setup.config_file.load() + fmt = config.get('pw', {}).get('pw_presubmit', {}).get('format', {}) + return FormatOptions( + python_formatter=fmt.get('python_formatter', 'yapf'), + black_path=fmt.get('black_path', 'black'), + # exclude=tuple(re.compile(x) for x in fmt.get('exclude', ())), + ) + + @dataclasses.dataclass class LuciPipeline: round: int @@ -503,12 +522,14 @@ class FormatContext: paths: Modified files for the presubmit step to check (often used in formatting steps but ignored in compile steps) package_root: Root directory for pw package installations + format_options: Formatting options, derived from pigweed.json """ root: Optional[Path] output_dir: Path paths: Tuple[Path, ...] package_root: Path + format_options: FormatOptions @dataclasses.dataclass @@ -530,6 +551,7 @@ class PresubmitContext: # pylint: disable=too-many-instance-attributes package_root: Root directory for pw package installations override_gn_args: Additional GN args processed by build.gn_gen() luci: Information about the LUCI build or None if not running in LUCI + format_options: Formatting options, derived from pigweed.json num_jobs: Number of jobs to run in parallel continue_after_build_error: For steps that compile, don't exit on the first compilation error @@ -544,6 +566,7 @@ class PresubmitContext: # pylint: disable=too-many-instance-attributes package_root: Path luci: Optional[LuciContext] override_gn_args: Dict[str, str] + format_options: FormatOptions num_jobs: Optional[int] = None continue_after_build_error: bool = False _failed: bool = False @@ -581,6 +604,7 @@ class PresubmitContext: # pylint: disable=too-many-instance-attributes package_root=root / 'environment' / 'packages', luci=None, override_gn_args={}, + format_options=FormatOptions(), ) @@ -859,6 +883,7 @@ class Presubmit: override_gn_args=self._override_gn_args, continue_after_build_error=self._continue_after_build_error, luci=LuciContext.create_from_environment(), + format_options=FormatOptions.load(), ) finally: @@ -1296,7 +1321,7 @@ class Check: _LOG.warning('%s', failure) return PresubmitResult.FAIL - except Exception as failure: # pylint: disable=broad-except + except Exception as _failure: # pylint: disable=broad-except _LOG.exception('Presubmit check %s failed!', self.name) return PresubmitResult.FAIL diff --git a/pw_presubmit/py/pw_presubmit/source_in_build.py b/pw_presubmit/py/pw_presubmit/source_in_build.py index e3df8345b..f19dd1174 100644 --- a/pw_presubmit/py/pw_presubmit/source_in_build.py +++ b/pw_presubmit/py/pw_presubmit/source_in_build.py @@ -11,7 +11,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations under # the License. -"""Check that source files are in the build.""" +"""Checks that source files are listed in build files, such as BUILD.bazel.""" import logging from typing import Callable, Sequence @@ -67,7 +67,7 @@ def bazel( for miss in missing: print(miss, file=outs) - _LOG.warning('All source files must appear in BUILD files') + _LOG.warning('All source files must appear in BUILD.bazel files') raise PresubmitFailure return source_is_in_bazel_build diff --git a/pw_presubmit/py/setup.cfg b/pw_presubmit/py/setup.cfg index 82dcf5508..5a71c4ff6 100644 --- a/pw_presubmit/py/setup.cfg +++ b/pw_presubmit/py/setup.cfg @@ -22,8 +22,8 @@ description = Presubmit tools and a presubmit script for Pigweed packages = find: zip_safe = False install_requires = - scan-build==2.0.19 - yapf==0.31.0 + yapf>=0.31.0 + black>=23.1.0 [options.package_data] pw_presubmit = py.typed diff --git a/pw_protobuf/BUILD.gn b/pw_protobuf/BUILD.gn index 8c21beb6a..97f8c82fe 100644 --- a/pw_protobuf/BUILD.gn +++ b/pw_protobuf/BUILD.gn @@ -120,8 +120,8 @@ pw_test_group("tests") { ":codegen_message_test", ":decoder_test", ":encoder_test", - ":encoder_fuzzer", - ":decoder_fuzzer", + ":encoder_fuzzer_test", + ":decoder_fuzzer_test", ":find_test", ":map_utils_test", ":message_test", @@ -131,6 +131,13 @@ pw_test_group("tests") { ] } +group("fuzzers") { + deps = [ + ":decoder_fuzzer", + ":encoder_fuzzer", + ] +} + pw_test("decoder_test") { deps = [ ":pw_protobuf" ] sources = [ "decoder_test.cc" ] @@ -252,12 +259,27 @@ pw_proto_library("codegen_test_protos") { ] } +# The tests below have a large amount of global and static data. +# TODO(b/234883746): Replace this with a better size-based check. +_small_executable_target_types = [ + "stm32f429i_executable", + "lm3s6965evb_executable", +] +_supports_large_tests = + _small_executable_target_types + [ pw_build_EXECUTABLE_TARGET_TYPE ] - + _small_executable_target_types != [] + pw_fuzzer("encoder_fuzzer") { sources = [ "encoder_fuzzer.cc", "fuzz.h", ] - deps = [ ":pw_protobuf" ] + deps = [ + ":pw_protobuf", + dir_pw_fuzzer, + dir_pw_span, + ] + enable_test_if = _supports_large_tests } pw_fuzzer("decoder_fuzzer") { @@ -265,5 +287,12 @@ pw_fuzzer("decoder_fuzzer") { "decoder_fuzzer.cc", "fuzz.h", ] - deps = [ ":pw_protobuf" ] + deps = [ + ":pw_protobuf", + dir_pw_fuzzer, + dir_pw_span, + dir_pw_status, + dir_pw_stream, + ] + enable_test_if = _supports_large_tests } diff --git a/pw_protobuf/CMakeLists.txt b/pw_protobuf/CMakeLists.txt index 4d1ecec9e..77d0ccf32 100644 --- a/pw_protobuf/CMakeLists.txt +++ b/pw_protobuf/CMakeLists.txt @@ -192,6 +192,11 @@ pw_proto_library(pw_protobuf.status_proto pw_protobuf_protos/status.proto ) +pw_proto_library(pw_protobuf.field_options_proto + SOURCES + pw_protobuf_protos/field_options.proto +) + pw_proto_library(pw_protobuf.codegen_protos SOURCES pw_protobuf_codegen_protos/codegen_options.proto diff --git a/pw_protobuf/decoder_fuzzer.cc b/pw_protobuf/decoder_fuzzer.cc index 1f70e9edd..7866b4381 100644 --- a/pw_protobuf/decoder_fuzzer.cc +++ b/pw_protobuf/decoder_fuzzer.cc @@ -27,10 +27,11 @@ #include "pw_stream/memory_stream.h" #include "pw_stream/stream.h" +namespace pw::protobuf::fuzz { namespace { void recursive_fuzzed_decode(FuzzedDataProvider& provider, - pw::protobuf::StreamDecoder& decoder, + StreamDecoder& decoder, uint32_t depth = 0) { constexpr size_t kMaxRepeatedRead = 1024; constexpr size_t kMaxDepth = 3; @@ -191,22 +192,28 @@ void recursive_fuzzed_decode(FuzzedDataProvider& provider, } } break; case kPush: { - pw::protobuf::StreamDecoder nested_decoder = decoder.GetNestedDecoder(); + StreamDecoder nested_decoder = decoder.GetNestedDecoder(); recursive_fuzzed_decode(provider, nested_decoder, depth + 1); } break; } } } -} // namespace -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - FuzzedDataProvider provider(data, size); +void TestOneInput(FuzzedDataProvider& provider) { constexpr size_t kMaxFuzzedProtoSize = 4096; std::vector<std::byte> proto_message_data = provider.ConsumeBytes<std::byte>( provider.ConsumeIntegralInRange<size_t>(0, kMaxFuzzedProtoSize)); - pw::stream::MemoryReader memory_reader(proto_message_data); - pw::protobuf::StreamDecoder decoder(memory_reader); + stream::MemoryReader memory_reader(proto_message_data); + StreamDecoder decoder(memory_reader); recursive_fuzzed_decode(provider, decoder); +} + +} // namespace +} // namespace pw::protobuf::fuzz + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + FuzzedDataProvider provider(data, size); + pw::protobuf::fuzz::TestOneInput(provider); return 0; } diff --git a/pw_protobuf/encoder_fuzzer.cc b/pw_protobuf/encoder_fuzzer.cc index d6163645a..ebd0984be 100644 --- a/pw_protobuf/encoder_fuzzer.cc +++ b/pw_protobuf/encoder_fuzzer.cc @@ -23,6 +23,7 @@ #include "pw_protobuf/encoder.h" #include "pw_span/span.h" +namespace pw::protobuf::fuzz { namespace { // TODO(b/235289495): Move this to pw_fuzzer/fuzzed_data_provider.h @@ -30,68 +31,63 @@ namespace { // Uses the given |provider| to pick and return a number between 0 and the // maximum numbers of T that can be generated from the remaining input data. template <typename T> -size_t ConsumeSize(FuzzedDataProvider* provider) { - size_t max = provider->remaining_bytes() / sizeof(T); - return provider->ConsumeIntegralInRange<size_t>(0, max); +size_t ConsumeSize(FuzzedDataProvider& provider) { + size_t max = provider.remaining_bytes() / sizeof(T); + return provider.ConsumeIntegralInRange<size_t>(0, max); } // Uses the given |provider| to generate several instances of T, store them in -// |data|, and then return a pw::span to them. It is the caller's responsbility -// to ensure |data| remains in scope as long as the returned pw::span. +// |data|, and then return a span to them. It is the caller's responsbility +// to ensure |data| remains in scope as long as the returned span. template <typename T> -pw::span<const T> ConsumeSpan(FuzzedDataProvider* provider, - std::vector<T>* data) { +span<const T> ConsumeSpan(FuzzedDataProvider& provider, std::vector<T>* data) { size_t num = ConsumeSize<T>(provider); size_t off = data->size(); data->reserve(off + num); for (size_t i = 0; i < num; ++i) { if constexpr (std::is_floating_point<T>::value) { - data->push_back(provider->ConsumeFloatingPoint<T>()); + data->push_back(provider.ConsumeFloatingPoint<T>()); } else { - data->push_back(provider->ConsumeIntegral<T>()); + data->push_back(provider.ConsumeIntegral<T>()); } } - return pw::span(&((*data)[off]), num); + return span(&((*data)[off]), num); } // Uses the given |provider| to generate a string, store it in |data|, and // return a C-style representation. It is the caller's responsbility to // ensure |data| remains in scope as long as the returned char*. -const char* ConsumeString(FuzzedDataProvider* provider, +const char* ConsumeString(FuzzedDataProvider& provider, std::vector<std::string>* data) { size_t off = data->size(); // OSS-Fuzz's clang doesn't have the zero-parameter version of // ConsumeRandomLengthString yet. size_t max_length = std::numeric_limits<size_t>::max(); - data->push_back(provider->ConsumeRandomLengthString(max_length)); + data->push_back(provider.ConsumeRandomLengthString(max_length)); return (*data)[off].c_str(); } // Uses the given |provider| to generate non-arithmetic bytes, store them in -// |data|, and return a pw::span to them. It is the caller's responsbility to -// ensure |data| remains in scope as long as the returned pw::span. -pw::span<const std::byte> ConsumeBytes(FuzzedDataProvider* provider, - std::vector<std::byte>* data) { +// |data|, and return a span to them. It is the caller's responsbility to +// ensure |data| remains in scope as long as the returned span. +span<const std::byte> ConsumeBytes(FuzzedDataProvider& provider, + std::vector<std::byte>* data) { size_t num = ConsumeSize<std::byte>(provider); - auto added = provider->ConsumeBytes<std::byte>(num); + auto added = provider.ConsumeBytes<std::byte>(num); size_t off = data->size(); num = added.size(); data->insert(data->end(), added.begin(), added.end()); - return pw::span(&((*data)[off]), num); + return span(&((*data)[off]), num); } -} // namespace - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { +void TestOneInput(FuzzedDataProvider& provider) { static std::byte buffer[65536]; - FuzzedDataProvider provider(data, size); - // Pick a subset of the buffer that the fuzzer is allowed to use, and poison // the rest. size_t unpoisoned_length = provider.ConsumeIntegralInRange<size_t>(0, sizeof(buffer)); - pw::span<std::byte> unpoisoned(buffer, unpoisoned_length); + ByteSpan unpoisoned(buffer, unpoisoned_length); void* poisoned = &buffer[unpoisoned_length]; size_t poisoned_length = sizeof(buffer) - unpoisoned_length; ASAN_POISON_MEMORY_REGION(poisoned, poisoned_length); @@ -119,163 +115,163 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { encoder .WriteUint32(provider.ConsumeIntegral<uint32_t>(), provider.ConsumeIntegral<uint32_t>()) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + .IgnoreError(); break; case kPackedUint32: encoder .WritePackedUint32(provider.ConsumeIntegral<uint32_t>(), - ConsumeSpan<uint32_t>(&provider, &u32s)) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + ConsumeSpan<uint32_t>(provider, &u32s)) + .IgnoreError(); break; case kUint64: encoder .WriteUint64(provider.ConsumeIntegral<uint32_t>(), provider.ConsumeIntegral<uint64_t>()) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + .IgnoreError(); break; case kPackedUint64: encoder .WritePackedUint64(provider.ConsumeIntegral<uint32_t>(), - ConsumeSpan<uint64_t>(&provider, &u64s)) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + ConsumeSpan<uint64_t>(provider, &u64s)) + .IgnoreError(); break; case kInt32: encoder .WriteInt32(provider.ConsumeIntegral<uint32_t>(), provider.ConsumeIntegral<int32_t>()) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + .IgnoreError(); break; case kPackedInt32: encoder .WritePackedInt32(provider.ConsumeIntegral<uint32_t>(), - ConsumeSpan<int32_t>(&provider, &s32s)) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + ConsumeSpan<int32_t>(provider, &s32s)) + .IgnoreError(); break; case kInt64: encoder .WriteInt64(provider.ConsumeIntegral<uint32_t>(), provider.ConsumeIntegral<int64_t>()) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + .IgnoreError(); break; case kPackedInt64: encoder .WritePackedInt64(provider.ConsumeIntegral<uint32_t>(), - ConsumeSpan<int64_t>(&provider, &s64s)) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + ConsumeSpan<int64_t>(provider, &s64s)) + .IgnoreError(); break; case kSint32: encoder .WriteSint32(provider.ConsumeIntegral<uint32_t>(), provider.ConsumeIntegral<int32_t>()) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + .IgnoreError(); break; case kPackedSint32: encoder .WritePackedSint32(provider.ConsumeIntegral<uint32_t>(), - ConsumeSpan<int32_t>(&provider, &s32s)) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + ConsumeSpan<int32_t>(provider, &s32s)) + .IgnoreError(); break; case kSint64: encoder .WriteSint64(provider.ConsumeIntegral<uint32_t>(), provider.ConsumeIntegral<int64_t>()) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + .IgnoreError(); break; case kPackedSint64: encoder .WritePackedSint64(provider.ConsumeIntegral<uint32_t>(), - ConsumeSpan<int64_t>(&provider, &s64s)) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + ConsumeSpan<int64_t>(provider, &s64s)) + .IgnoreError(); break; case kBool: encoder .WriteBool(provider.ConsumeIntegral<uint32_t>(), provider.ConsumeBool()) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + .IgnoreError(); break; case kFixed32: encoder .WriteFixed32(provider.ConsumeIntegral<uint32_t>(), provider.ConsumeIntegral<uint32_t>()) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + .IgnoreError(); break; case kPackedFixed32: encoder .WritePackedFixed32(provider.ConsumeIntegral<uint32_t>(), - ConsumeSpan<uint32_t>(&provider, &u32s)) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + ConsumeSpan<uint32_t>(provider, &u32s)) + .IgnoreError(); break; case kFixed64: encoder .WriteFixed64(provider.ConsumeIntegral<uint32_t>(), provider.ConsumeIntegral<uint64_t>()) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + .IgnoreError(); break; case kPackedFixed64: encoder .WritePackedFixed64(provider.ConsumeIntegral<uint32_t>(), - ConsumeSpan<uint64_t>(&provider, &u64s)) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + ConsumeSpan<uint64_t>(provider, &u64s)) + .IgnoreError(); break; case kSfixed32: encoder .WriteSfixed32(provider.ConsumeIntegral<uint32_t>(), provider.ConsumeIntegral<int32_t>()) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + .IgnoreError(); break; case kPackedSfixed32: encoder .WritePackedSfixed32(provider.ConsumeIntegral<uint32_t>(), - ConsumeSpan<int32_t>(&provider, &s32s)) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + ConsumeSpan<int32_t>(provider, &s32s)) + .IgnoreError(); break; case kSfixed64: encoder .WriteSfixed64(provider.ConsumeIntegral<uint32_t>(), provider.ConsumeIntegral<int64_t>()) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + .IgnoreError(); break; case kPackedSfixed64: encoder .WritePackedSfixed64(provider.ConsumeIntegral<uint32_t>(), - ConsumeSpan<int64_t>(&provider, &s64s)) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + ConsumeSpan<int64_t>(provider, &s64s)) + .IgnoreError(); break; case kFloat: encoder .WriteFloat(provider.ConsumeIntegral<uint32_t>(), provider.ConsumeFloatingPoint<float>()) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + .IgnoreError(); break; case kPackedFloat: encoder .WritePackedFloat(provider.ConsumeIntegral<uint32_t>(), - ConsumeSpan<float>(&provider, &floats)) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + ConsumeSpan<float>(provider, &floats)) + .IgnoreError(); break; case kDouble: encoder .WriteDouble(provider.ConsumeIntegral<uint32_t>(), provider.ConsumeFloatingPoint<double>()) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + .IgnoreError(); break; case kPackedDouble: encoder .WritePackedDouble(provider.ConsumeIntegral<uint32_t>(), - ConsumeSpan<double>(&provider, &doubles)) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + ConsumeSpan<double>(provider, &doubles)) + .IgnoreError(); break; case kBytes: encoder .WriteBytes(provider.ConsumeIntegral<uint32_t>(), - ConsumeBytes(&provider, &bytes)) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + ConsumeBytes(provider, &bytes)) + .IgnoreError(); break; case kString: encoder .WriteString(provider.ConsumeIntegral<uint32_t>(), - ConsumeString(&provider, &strings)) - .IgnoreError(); // TODO(b/242598609): Handle Status properly + ConsumeString(provider, &strings)) + .IgnoreError(); break; case kPush: // Special "field". The marks the start of a nested message. @@ -286,5 +282,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { // Don't forget to unpoison for the next iteration! ASAN_UNPOISON_MEMORY_REGION(poisoned, poisoned_length); +} + +} // namespace +} // namespace pw::protobuf::fuzz + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + FuzzedDataProvider provider(data, size); + pw::protobuf::fuzz::TestOneInput(provider); return 0; } diff --git a/pw_protobuf/fuzz.h b/pw_protobuf/fuzz.h index 78dc34a73..cc95db65c 100644 --- a/pw_protobuf/fuzz.h +++ b/pw_protobuf/fuzz.h @@ -13,7 +13,9 @@ // the License. #pragma once -namespace { +#include <cstdint> + +namespace pw::protobuf::fuzz { // Encodable values. The fuzzer will iteratively choose different field types to // generate and encode. @@ -49,4 +51,4 @@ enum FieldType : uint8_t { kMaxValue = kPush, }; -} // namespace +} // namespace pw::protobuf::fuzz diff --git a/pw_protobuf/py/pw_protobuf/codegen_pwpb.py b/pw_protobuf/py/pw_protobuf/codegen_pwpb.py index 196a029c2..dd862645b 100644 --- a/pw_protobuf/py/pw_protobuf/codegen_pwpb.py +++ b/pw_protobuf/py/pw_protobuf/codegen_pwpb.py @@ -2467,6 +2467,24 @@ def generate_table_for_message( f'inline constexpr pw::span<const {_INTERNAL_NAMESPACE}::' 'MessageField> kMessageFields = _kMessageFields;' ) + + member_list = ', '.join( + [f'message.{prop.struct_member()[1]}' for prop in properties] + ) + + # Generate std::tuple for Message fields. + output.write_line( + 'inline constexpr auto ToTuple(const Message &message) {' + ) + output.write_line(f' return std::tie({member_list});') + output.write_line('}') + + # Generate mutable std::tuple for Message fields. + output.write_line( + 'inline constexpr auto ToMutableTuple(Message &message) {' + ) + output.write_line(f' return std::tie({member_list});') + output.write_line('}') else: output.write_line( f'inline constexpr pw::span<const {_INTERNAL_NAMESPACE}::' diff --git a/pw_protobuf/py/setup.cfg b/pw_protobuf/py/setup.cfg index 8c2cc2370..883edded4 100644 --- a/pw_protobuf/py/setup.cfg +++ b/pw_protobuf/py/setup.cfg @@ -22,8 +22,8 @@ description = Lightweight streaming protobuf implementation packages = find: zip_safe = False install_requires = - protobuf==3.20.1 - googleapis-common-protos==1.56.2 + protobuf>=3.20.1 + googleapis-common-protos>=1.56.2 graphlib-backport;python_version<'3.9' [options.entry_points] diff --git a/pw_protobuf_compiler/proto.cmake b/pw_protobuf_compiler/proto.cmake index 4c518abfe..8ed7519aa 100644 --- a/pw_protobuf_compiler/proto.cmake +++ b/pw_protobuf_compiler/proto.cmake @@ -221,9 +221,11 @@ function(_pw_generate_protos TARGET LANGUAGE) set(generated_outputs "${outputs}" PARENT_SCOPE) if("${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "Windows") - get_filename_component(dir "${source_file}" DIRECTORY) - get_filename_component(name "${source_file}" NAME_WE) - set(arg_PLUGIN "${dir}/${name}.bat") + foreach(source_file IN LISTS SOURCES) + get_filename_component(dir "${source_file}" DIRECTORY) + get_filename_component(name "${source_file}" NAME_WE) + set(arg_PLUGIN "${dir}/${name}.bat") + endforeach() endif() set(script "$ENV{PW_ROOT}/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_protos.py") diff --git a/pw_protobuf_compiler/py/pw_protobuf_compiler/python_protos.py b/pw_protobuf_compiler/py/pw_protobuf_compiler/python_protos.py index 729e3c580..5d379cbb6 100644 --- a/pw_protobuf_compiler/py/pw_protobuf_compiler/python_protos.py +++ b/pw_protobuf_compiler/py/pw_protobuf_compiler/python_protos.py @@ -36,36 +36,16 @@ from typing import ( Union, ) -# Temporarily set the root logger level to critical while importing yapf. -# This silences INFO level messages from -# environment/cipd/packages/python/lib/python3.9/lib2to3/driver.py -# when it writes Grammar3.*.pickle and PatternGrammar3.*.pickle files. -_original_level = 0 -for handler in logging.getLogger().handlers: - # pylint: disable=unidiomatic-typecheck - if type(handler) == logging.StreamHandler: - if handler.level > _original_level: - _original_level = handler.level - handler.level = logging.CRITICAL - # pylint: enable=unidiomatic-typecheck - try: # pylint: disable=wrong-import-position - from yapf.yapflib import yapf_api # type: ignore[import] + import black + + black_mode: Optional[black.Mode] = black.Mode(string_normalization=False) # pylint: enable=wrong-import-position except ImportError: - yapf_api = None - -# Restore the original stderr/out log handler level. -for handler in logging.getLogger().handlers: - # Must use type() check here since isinstance returns True for FileHandlers - # and StreamHandler: isinstance(logging.FileHandler, logging.StreamHandler) - # pylint: disable=unidiomatic-typecheck - if type(handler) == logging.StreamHandler: - handler.level = _original_level - # pylint: enable=unidiomatic-typecheck -del _original_level + black = None # type: ignore + black_mode = None _LOG = logging.getLogger(__name__) @@ -471,12 +451,12 @@ def proto_repr(message, *, wrap: bool = True) -> str: Args: message: The protobuf message to format - wrap: If true and YAPF is available, the output is wrapped according to - PEP8 using YAPF. + wrap: If true and black is available, the output is wrapped according to + PEP8 using black. """ raw = f'{message.DESCRIPTOR.full_name}({", ".join(_proto_repr(message))})' - if wrap and yapf_api is not None: - return yapf_api.FormatCode(raw, style_config='PEP8')[0].rstrip() + if wrap and black is not None and black_mode is not None: + return black.format_str(raw, mode=black_mode).strip() return raw diff --git a/pw_protobuf_compiler/py/python_protos_test.py b/pw_protobuf_compiler/py/python_protos_test.py index 1c105f7c6..819bede33 100755 --- a/pw_protobuf_compiler/py/python_protos_test.py +++ b/pw_protobuf_compiler/py/python_protos_test.py @@ -448,10 +448,12 @@ class TestProtoRepr(unittest.TestCase): def test_wrap_multiple_lines(self): self.assertEqual( """\ -pw.test3.Message(optional_int=0, - optional_bytes=b'', - optional_string='', - optional_enum=pw.test3.Enum.ZERO)""", +pw.test3.Message( + optional_int=0, + optional_bytes=b'', + optional_string='', + optional_enum=pw.test3.Enum.ZERO, +)""", proto_repr( self.message( optional_int=0, diff --git a/pw_protobuf_compiler/py/setup.cfg b/pw_protobuf_compiler/py/setup.cfg index 98b45f630..8e3a6eca0 100644 --- a/pw_protobuf_compiler/py/setup.cfg +++ b/pw_protobuf_compiler/py/setup.cfg @@ -22,11 +22,13 @@ description = Pigweed protoc wrapper packages = find: zip_safe = False install_requires = - # NOTE: mypy needs to stay in sync with mypy-protobuf - # Currently using mypy 0.991 and mypy-protobuf 3.3.0 (see constraint.list) + # NOTE: protobuf needs to stay in sync with mypy-protobuf + # Currently using mypy protobuf 3.20.1 and mypy-protobuf 3.3.0 (see + # constraint.list). These requirements should stay as >= the lowest version + # we support. mypy-protobuf>=3.2.0 - protobuf==3.20.1 - types-protobuf==3.19.22 + protobuf>=3.20.1 + types-protobuf>=3.19.22 [options.package_data] pw_protobuf_compiler = py.typed diff --git a/pw_random/BUILD.gn b/pw_random/BUILD.gn index 4b2a5c44a..0563fc2e2 100644 --- a/pw_random/BUILD.gn +++ b/pw_random/BUILD.gn @@ -48,10 +48,14 @@ pw_source_set("fuzzer_generator") { pw_test_group("tests") { tests = [ ":xor_shift_star_test", - ":get_int_bounded_fuzzer", + ":get_int_bounded_fuzzer_test", ] } +group("fuzzers") { + deps = [ ":get_int_bounded_fuzzer" ] +} + pw_test("xor_shift_star_test") { deps = [ ":pw_random", diff --git a/pw_rpc/BUILD.gn b/pw_rpc/BUILD.gn index ad7ff7b1e..18c8778e3 100644 --- a/pw_rpc/BUILD.gn +++ b/pw_rpc/BUILD.gn @@ -455,6 +455,7 @@ pw_test_group("tests") { ":service_test", ] group_deps = [ + "fuzz:tests", "nanopb:tests", "pwpb:tests", "raw:tests", diff --git a/pw_rpc/benchmark.rst b/pw_rpc/benchmark.rst index 6e97ac1d0..5ecc136a8 100644 --- a/pw_rpc/benchmark.rst +++ b/pw_rpc/benchmark.rst @@ -49,3 +49,23 @@ Example server.RegisterService(benchmark_service); } +Stress Test +=========== +.. attention:: + This section is experimental and liable to change. + +The Benchmark service is also used as part of a stress test of the ``pw_rpc`` +module. This stress test is implemented as an unguided fuzzer that uses +multiple worker threads to perform generated sequences of actions using RPC +``Call`` objects. The test is included as an integration test, and can found and +be run locally using GN: + +.. code-block:: bash + + $ gn desc out //:integration_tests deps | grep fuzz + //pw_rpc/fuzz:cpp_client_server_fuzz_test(//targets/host/pigweed_internal:pw_strict_host_clang_debug) + + $ gn outputs out '//pw_rpc/fuzz:cpp_client_server_fuzz_test(//targets/host/pigweed_internal:pw_strict_host_clang_debug)' + pw_strict_host_clang_debug/gen/pw_rpc/fuzz/cpp_client_server_fuzz_test.pw_pystamp + + $ ninja -C out pw_strict_host_clang_debug/gen/pw_rpc/fuzz/cpp_client_server_fuzz_test.pw_pystamp diff --git a/pw_rpc/client.cc b/pw_rpc/client.cc index 4b5fc5cf0..006264220 100644 --- a/pw_rpc/client.cc +++ b/pw_rpc/client.cc @@ -87,9 +87,7 @@ Status Client::ProcessPacket(ConstByteSpan data) { case PacketType::REQUEST: case PacketType::CLIENT_STREAM: - case PacketType::DEPRECATED_SERVER_STREAM_END: case PacketType::CLIENT_ERROR: - case PacketType::DEPRECATED_CANCEL: case PacketType::CLIENT_STREAM_END: default: internal::rpc_lock().unlock(); diff --git a/pw_rpc/fake_channel_output.cc b/pw_rpc/fake_channel_output.cc index d6520145b..6404e68d6 100644 --- a/pw_rpc/fake_channel_output.cc +++ b/pw_rpc/fake_channel_output.cc @@ -70,8 +70,6 @@ Status FakeChannelOutput::HandlePacket(span<const std::byte> buffer) { return OkStatus(); case pwpb::PacketType::CLIENT_STREAM: return OkStatus(); - case pwpb::PacketType::DEPRECATED_SERVER_STREAM_END: - PW_CRASH("Deprecated PacketType %d", static_cast<int>(packet.type())); case pwpb::PacketType::CLIENT_ERROR: PW_LOG_WARN("FakeChannelOutput received client error: %s", packet.status().str()); @@ -80,7 +78,6 @@ Status FakeChannelOutput::HandlePacket(span<const std::byte> buffer) { PW_LOG_WARN("FakeChannelOutput received server error: %s", packet.status().str()); return OkStatus(); - case pwpb::PacketType::DEPRECATED_CANCEL: case pwpb::PacketType::SERVER_STREAM: case pwpb::PacketType::CLIENT_STREAM_END: return OkStatus(); diff --git a/pw_rpc/fuzz/BUILD.gn b/pw_rpc/fuzz/BUILD.gn new file mode 100644 index 000000000..bec42fe44 --- /dev/null +++ b/pw_rpc/fuzz/BUILD.gn @@ -0,0 +1,140 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import("//build_overrides/pigweed.gni") + +import("$dir_pw_build/target_types.gni") +import("$dir_pw_chrono/backend.gni") +import("$dir_pw_rpc/internal/integration_test_ports.gni") +import("$dir_pw_thread/backend.gni") +import("$dir_pw_unit_test/test.gni") + +config("public_include_path") { + include_dirs = [ + "public", + "$dir_pw_rpc/public", + ] + visibility = [ ":*" ] +} + +pw_source_set("alarm_timer") { + public_configs = [ ":public_include_path" ] + public = [ "public/pw_rpc/fuzz/alarm_timer.h" ] + public_deps = [ + "$dir_pw_chrono:system_clock", + "$dir_pw_chrono:system_timer", + ] + visibility = [ ":*" ] +} + +pw_test("alarm_timer_test") { + enable_if = pw_chrono_SYSTEM_TIMER_BACKEND != "" + sources = [ "alarm_timer_test.cc" ] + deps = [ + ":alarm_timer", + "$dir_pw_sync:binary_semaphore", + ] +} + +pw_source_set("argparse") { + public_configs = [ ":public_include_path" ] + public = [ "public/pw_rpc/fuzz/argparse.h" ] + sources = [ "argparse.cc" ] + public_deps = [ + "$dir_pw_containers:vector", + dir_pw_status, + ] + deps = [ + "$dir_pw_string:builder", + dir_pw_assert, + dir_pw_log, + ] + visibility = [ ":*" ] +} + +pw_test("argparse_test") { + sources = [ "argparse_test.cc" ] + deps = [ ":argparse" ] +} + +pw_source_set("engine") { + public_configs = [ ":public_include_path" ] + public = [ "public/pw_rpc/fuzz/engine.h" ] + sources = [ "engine.cc" ] + public_deps = [ + ":alarm_timer", + "$dir_pw_chrono:system_clock", + "$dir_pw_rpc:benchmark", + "$dir_pw_rpc:log_config", + "$dir_pw_rpc:protos.raw_rpc", + "$dir_pw_string:format", + "$dir_pw_sync:condition_variable", + "$dir_pw_sync:timed_mutex", + "$dir_pw_thread:thread", + dir_pw_random, + ] + deps = [ "$dir_pw_rpc:client" ] + visibility = [ ":*" ] +} + +pw_test("engine_test") { + enable_if = + pw_chrono_SYSTEM_TIMER_BACKEND == "$dir_pw_chrono_stl:system_timer" && + pw_thread_THREAD_BACKEND == "$dir_pw_thread_stl:thread" + sources = [ "engine_test.cc" ] + deps = [ + ":engine", + "$dir_pw_rpc:client_server_testing_threaded", + "$dir_pw_thread:test_threads", + "$dir_pw_thread_stl:test_threads", + dir_pw_log, + pw_chrono_SYSTEM_TIMER_BACKEND, + ] +} + +pw_executable("client_fuzzer") { + sources = [ "client_fuzzer.cc" ] + deps = [ + ":argparse", + ":engine", + "$dir_pw_rpc:client", + "$dir_pw_rpc:integration_testing", + ] +} + +pw_python_action("cpp_client_server_fuzz_test") { + script = "../py/pw_rpc/testing.py" + args = [ + "--server", + "<TARGET_FILE($dir_pw_rpc:test_rpc_server)>", + "--client", + "<TARGET_FILE(:client_fuzzer)>", + "--", + "$pw_rpc_CPP_CLIENT_FUZZER_TEST_PORT", + ] + deps = [ + ":client_fuzzer", + "$dir_pw_rpc:test_rpc_server", + ] + + stamp = true +} + +pw_test_group("tests") { + tests = [ + ":argparse_test", + ":alarm_timer_test", + ":engine_test", + ] +} diff --git a/pw_rpc/fuzz/alarm_timer_test.cc b/pw_rpc/fuzz/alarm_timer_test.cc new file mode 100644 index 000000000..ce8395a6b --- /dev/null +++ b/pw_rpc/fuzz/alarm_timer_test.cc @@ -0,0 +1,64 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#include "pw_rpc/fuzz/alarm_timer.h" + +#include <chrono> + +#include "gtest/gtest.h" +#include "pw_sync/binary_semaphore.h" + +namespace pw::rpc::fuzz { +namespace { + +using namespace std::chrono_literals; + +TEST(AlarmTimerTest, Start) { + sync::BinarySemaphore sem; + AlarmTimer timer([&sem](chrono::SystemClock::time_point) { sem.release(); }); + timer.Start(10ms); + sem.acquire(); +} + +TEST(AlarmTimerTest, Restart) { + sync::BinarySemaphore sem; + AlarmTimer timer([&sem](chrono::SystemClock::time_point) { sem.release(); }); + timer.Start(50ms); + for (size_t i = 0; i < 10; ++i) { + timer.Restart(); + EXPECT_FALSE(sem.try_acquire_for(10us)); + } + sem.acquire(); +} + +TEST(AlarmTimerTest, Cancel) { + sync::BinarySemaphore sem; + AlarmTimer timer([&sem](chrono::SystemClock::time_point) { sem.release(); }); + timer.Start(50ms); + timer.Cancel(); + EXPECT_FALSE(sem.try_acquire_for(100us)); +} + +TEST(AlarmTimerTest, Destroy) { + sync::BinarySemaphore sem; + { + AlarmTimer timer( + [&sem](chrono::SystemClock::time_point) { sem.release(); }); + timer.Start(50ms); + } + EXPECT_FALSE(sem.try_acquire_for(100us)); +} + +} // namespace +} // namespace pw::rpc::fuzz diff --git a/pw_rpc/fuzz/argparse.cc b/pw_rpc/fuzz/argparse.cc new file mode 100644 index 000000000..39a5dd694 --- /dev/null +++ b/pw_rpc/fuzz/argparse.cc @@ -0,0 +1,259 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#include "pw_rpc/fuzz/argparse.h" + +#include <cctype> +#include <cstring> + +#include "pw_assert/check.h" +#include "pw_log/log.h" +#include "pw_string/string_builder.h" + +namespace pw::rpc::fuzz { +namespace { + +// Visitor to `ArgVariant` used by `ParseArgs` below. +struct ParseVisitor { + std::string_view arg0; + std::string_view arg1; + + template <typename Parser> + ParseStatus operator()(Parser& parser) { + return parser.Parse(arg0, arg1); + } +}; + +// Visitor to `ArgVariant` used by `GetArg` below. +struct ValueVisitor { + std::string_view name; + + template <typename Parser> + std::optional<ArgVariant> operator()(Parser& parser) { + std::optional<ArgVariant> result; + if (parser.short_name() == name || parser.long_name() == name) { + result.emplace(parser.value()); + } + return result; + } +}; + +// Visitor to `ArgVariant` used by `PrintUsage` below. +const size_t kMaxUsageLen = 256; +struct UsageVisitor { + StringBuffer<kMaxUsageLen>* buffer; + + void operator()(const BoolParser& parser) const { + auto short_name = parser.short_name(); + auto long_name = parser.long_name(); + *buffer << " [" << short_name << "|--[no-]" << long_name.substr(2) << "]"; + } + + template <typename T> + void operator()(const UnsignedParser<T>& parser) const { + auto short_name = parser.short_name(); + auto long_name = parser.long_name(); + *buffer << " "; + if (!parser.positional()) { + *buffer << "["; + if (!short_name.empty()) { + *buffer << short_name << "|"; + } + *buffer << long_name << " "; + } + for (const auto& c : long_name) { + *buffer << static_cast<char>(toupper(c)); + } + if (!parser.positional()) { + *buffer << "]"; + } + } +}; + +// Visitor to `ArgVariant` used by `ResetArg` below. +struct ResetVisitor { + std::string_view name; + + template <typename Parser> + bool operator()(Parser& parser) { + if (parser.short_name() != name && parser.long_name() != name) { + return false; + } + parser.Reset(); + return true; + } +}; + +} // namespace + +ArgParserBase::ArgParserBase(std::string_view name) : long_name_(name) { + PW_CHECK(!name.empty()); + PW_CHECK(name != "--"); + positional_ = + name[0] != '-' || (name.size() > 2 && name.substr(0, 2) != "--"); +} + +ArgParserBase::ArgParserBase(std::string_view shortopt, + std::string_view longopt) + : short_name_(shortopt), long_name_(longopt) { + PW_CHECK(shortopt.size() == 2); + PW_CHECK(shortopt[0] == '-'); + PW_CHECK(shortopt != "--"); + PW_CHECK(longopt.size() > 2); + PW_CHECK(longopt.substr(0, 2) == "--"); + positional_ = false; +} + +bool ArgParserBase::Match(std::string_view arg) { + if (arg.empty()) { + return false; + } + if (!positional_) { + return arg == short_name_ || arg == long_name_; + } + if (!std::holds_alternative<std::monostate>(value_)) { + return false; + } + if ((arg.size() == 2 && arg[0] == '-') || + (arg.size() > 2 && arg.substr(0, 2) == "--")) { + PW_LOG_WARN("Argument parsed for '%s' appears to be a flag: '%s'", + long_name_.data(), + arg.data()); + } + return true; +} + +const ArgVariant& ArgParserBase::GetValue() const { + return std::holds_alternative<std::monostate>(value_) ? initial_ : value_; +} + +BoolParser::BoolParser(std::string_view name) : ArgParserBase(name) {} +BoolParser::BoolParser(std::string_view shortopt, std::string_view longopt) + : ArgParserBase(shortopt, longopt) {} + +BoolParser& BoolParser::set_default(bool value) { + set_initial(value); + return *this; +} + +ParseStatus BoolParser::Parse(std::string_view arg0, + [[maybe_unused]] std::string_view arg1) { + if (Match(arg0)) { + set_value(true); + return kParsedOne; + } + if (arg0.size() > 5 && arg0.substr(0, 5) == "--no-" && + arg0.substr(5) == long_name().substr(2)) { + set_value(false); + return kParsedOne; + } + return kParseMismatch; +} + +UnsignedParserBase::UnsignedParserBase(std::string_view name) + : ArgParserBase(name) {} +UnsignedParserBase::UnsignedParserBase(std::string_view shortopt, + std::string_view longopt) + : ArgParserBase(shortopt, longopt) {} + +ParseStatus UnsignedParserBase::Parse(std::string_view arg0, + std::string_view arg1, + uint64_t max) { + auto result = kParsedOne; + if (!Match(arg0)) { + return kParseMismatch; + } + if (!positional()) { + if (arg1.empty()) { + PW_LOG_ERROR("Missing value for flag '%s'", arg0.data()); + return kParseFailure; + } + arg0 = arg1; + result = kParsedTwo; + } + char* endptr; + auto value = strtoull(arg0.data(), &endptr, 0); + if (*endptr) { + PW_LOG_ERROR("Failed to parse number from '%s'", arg0.data()); + return kParseFailure; + } + if (value > max) { + PW_LOG_ERROR("Parsed value is too large: %llu", value); + return kParseFailure; + } + set_value(value); + return result; +} + +Status ParseArgs(Vector<ArgParserVariant>& parsers, int argc, char** argv) { + for (int i = 1; i < argc; ++i) { + auto arg0 = std::string_view(argv[i]); + auto arg1 = + i == (argc - 1) ? std::string_view() : std::string_view(argv[i + 1]); + bool parsed = false; + for (auto& parser : parsers) { + switch (std::visit(ParseVisitor{.arg0 = arg0, .arg1 = arg1}, parser)) { + case kParsedOne: + break; + case kParsedTwo: + ++i; + break; + case kParseMismatch: + continue; + case kParseFailure: + PW_LOG_ERROR("Failed to parse '%s'", arg0.data()); + return Status::InvalidArgument(); + } + parsed = true; + break; + } + if (!parsed) { + PW_LOG_ERROR("Unrecognized argument: '%s'", arg0.data()); + return Status::InvalidArgument(); + } + } + return OkStatus(); +} + +void PrintUsage(const Vector<ArgParserVariant>& parsers, + std::string_view argv0) { + StringBuffer<kMaxUsageLen> buffer; + buffer << "usage: " << argv0; + for (auto& parser : parsers) { + std::visit(UsageVisitor{.buffer = &buffer}, parser); + } + PW_LOG_INFO("%s", buffer.c_str()); +} + +std::optional<ArgVariant> GetArg(const Vector<ArgParserVariant>& parsers, + std::string_view name) { + for (auto& parser : parsers) { + if (auto result = std::visit(ValueVisitor{.name = name}, parser); + result.has_value()) { + return result; + } + } + return std::optional<ArgVariant>(); +} + +Status ResetArg(Vector<ArgParserVariant>& parsers, std::string_view name) { + for (auto& parser : parsers) { + if (std::visit(ResetVisitor{.name = name}, parser)) { + return OkStatus(); + } + } + return Status::InvalidArgument(); +} + +} // namespace pw::rpc::fuzz diff --git a/pw_rpc/fuzz/argparse_test.cc b/pw_rpc/fuzz/argparse_test.cc new file mode 100644 index 000000000..6f0ab3889 --- /dev/null +++ b/pw_rpc/fuzz/argparse_test.cc @@ -0,0 +1,196 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#include "pw_rpc/fuzz/argparse.h" + +#include <cstdint> +#include <limits> + +#include "gtest/gtest.h" + +namespace pw::rpc::fuzz { +namespace { + +TEST(ArgsParseTest, ParseBoolFlag) { + auto parser1 = BoolParser("-t", "--true").set_default(true); + auto parser2 = BoolParser("-f").set_default(false); + EXPECT_TRUE(parser1.value()); + EXPECT_FALSE(parser2.value()); + + EXPECT_EQ(parser1.Parse("-t"), ParseStatus::kParsedOne); + EXPECT_EQ(parser2.Parse("-t"), ParseStatus::kParseMismatch); + EXPECT_TRUE(parser1.value()); + EXPECT_FALSE(parser2.value()); + + EXPECT_EQ(parser1.Parse("--true"), ParseStatus::kParsedOne); + EXPECT_EQ(parser2.Parse("--true"), ParseStatus::kParseMismatch); + EXPECT_TRUE(parser1.value()); + EXPECT_FALSE(parser2.value()); + + EXPECT_EQ(parser1.Parse("--no-true"), ParseStatus::kParsedOne); + EXPECT_EQ(parser2.Parse("--no-true"), ParseStatus::kParseMismatch); + EXPECT_FALSE(parser1.value()); + EXPECT_FALSE(parser2.value()); + + EXPECT_EQ(parser1.Parse("-f"), ParseStatus::kParseMismatch); + EXPECT_EQ(parser2.Parse("-f"), ParseStatus::kParsedOne); + EXPECT_FALSE(parser1.value()); + EXPECT_TRUE(parser2.value()); +} + +template <typename T> +void ParseUnsignedFlag() { + auto parser = UnsignedParser<T>("-u", "--unsigned").set_default(137); + EXPECT_EQ(parser.value(), 137u); + + // Wrong name. + EXPECT_EQ(parser.Parse("-s"), ParseStatus::kParseMismatch); + EXPECT_EQ(parser.Parse("--signed"), ParseStatus::kParseMismatch); + EXPECT_EQ(parser.value(), 137u); + + // Missing values. + EXPECT_EQ(parser.Parse("-u"), ParseStatus::kParseFailure); + EXPECT_EQ(parser.Parse("--unsigned"), ParseStatus::kParseFailure); + EXPECT_EQ(parser.value(), 137u); + + // Non-numeric values. + EXPECT_EQ(parser.Parse("-u", "foo"), ParseStatus::kParseFailure); + EXPECT_EQ(parser.Parse("--unsigned", "bar"), ParseStatus::kParseFailure); + EXPECT_EQ(parser.value(), 137u); + + // Minimum values. + EXPECT_EQ(parser.Parse("-u", "0"), ParseStatus::kParsedTwo); + EXPECT_EQ(parser.Parse("--unsigned", "0"), ParseStatus::kParsedTwo); + EXPECT_EQ(parser.value(), 0u); + + // Maximum values. + T max = std::numeric_limits<T>::max(); + StringBuffer<32> buf; + buf << max; + EXPECT_EQ(parser.Parse("-u", buf.c_str()), ParseStatus::kParsedTwo); + EXPECT_EQ(parser.value(), max); + EXPECT_EQ(parser.Parse("--unsigned", buf.c_str()), ParseStatus::kParsedTwo); + EXPECT_EQ(parser.value(), max); + + // Out of-range value. + if (max < std::numeric_limits<uint64_t>::max()) { + buf.clear(); + buf << (max + 1ULL); + EXPECT_EQ(parser.Parse("-u", buf.c_str()), ParseStatus::kParseFailure); + EXPECT_EQ(parser.Parse("--unsigned", buf.c_str()), + ParseStatus::kParseFailure); + EXPECT_EQ(parser.value(), max); + } +} + +TEST(ArgsParseTest, ParseUnsignedFlags) { + ParseUnsignedFlag<uint8_t>(); + ParseUnsignedFlag<uint16_t>(); + ParseUnsignedFlag<uint32_t>(); + ParseUnsignedFlag<uint64_t>(); +} + +TEST(ArgsParseTest, ParsePositional) { + auto parser = UnsignedParser<size_t>("positional").set_default(1); + EXPECT_EQ(parser.Parse("-p", "2"), ParseStatus::kParseFailure); + EXPECT_EQ(parser.value(), 1u); + + EXPECT_EQ(parser.Parse("--positional", "2"), ParseStatus::kParseFailure); + EXPECT_EQ(parser.value(), 1u); + + // Second arg is ignored.. + EXPECT_EQ(parser.Parse("2", "3"), ParseStatus::kParsedOne); + EXPECT_EQ(parser.value(), 2u); + + // Positional only matches once. + EXPECT_EQ(parser.Parse("3"), ParseStatus::kParseMismatch); + EXPECT_EQ(parser.value(), 2u); +} + +TEST(ArgsParseTest, PrintUsage) { + // Just verify it compiles and runs. + Vector<ArgParserVariant, 3> parsers = { + BoolParser("-v", "--verbose").set_default(false), + UnsignedParser<size_t>("-r", "--runs").set_default(1000), + UnsignedParser<size_t>("port").set_default(11111), + }; + PrintUsage(parsers, "test-bin"); +} + +void CheckArgs(Vector<ArgParserVariant>& parsers, + bool verbose, + size_t runs, + uint16_t port) { + bool actual_verbose; + EXPECT_EQ(GetArg(parsers, "--verbose", &actual_verbose), OkStatus()); + EXPECT_EQ(verbose, actual_verbose); + EXPECT_EQ(ResetArg(parsers, "--verbose"), OkStatus()); + + size_t actual_runs; + EXPECT_EQ(GetArg(parsers, "--runs", &actual_runs), OkStatus()); + EXPECT_EQ(runs, actual_runs); + EXPECT_EQ(ResetArg(parsers, "--runs"), OkStatus()); + + uint16_t actual_port; + EXPECT_EQ(GetArg(parsers, "port", &actual_port), OkStatus()); + EXPECT_EQ(port, actual_port); + EXPECT_EQ(ResetArg(parsers, "port"), OkStatus()); +} + +TEST(ArgsParseTest, ParseArgs) { + Vector<ArgParserVariant, 3> parsers{ + BoolParser("-v", "--verbose").set_default(false), + UnsignedParser<size_t>("-r", "--runs").set_default(1000), + UnsignedParser<uint16_t>("port").set_default(11111), + }; + + char const* argv1[] = {"test-bin"}; + EXPECT_EQ(ParseArgs(parsers, 1, const_cast<char**>(argv1)), OkStatus()); + CheckArgs(parsers, false, 1000, 11111); + + char const* argv2[] = {"test-bin", "22222"}; + EXPECT_EQ(ParseArgs(parsers, 2, const_cast<char**>(argv2)), OkStatus()); + CheckArgs(parsers, false, 1000, 22222); + + // Out of range argument. + char const* argv3[] = {"test-bin", "65536"}; + EXPECT_EQ(ParseArgs(parsers, 2, const_cast<char**>(argv3)), + Status::InvalidArgument()); + + // Extra argument. + char const* argv4[] = {"test-bin", "1", "2"}; + EXPECT_EQ(ParseArgs(parsers, 3, const_cast<char**>(argv4)), + Status::InvalidArgument()); + EXPECT_EQ(ResetArg(parsers, "port"), OkStatus()); + + // Flag missing value. + char const* argv5[] = {"test-bin", "--runs"}; + EXPECT_EQ(ParseArgs(parsers, 2, const_cast<char**>(argv5)), + Status::InvalidArgument()); + + char const* argv6[] = {"test-bin", "-v", "33333", "--runs", "300"}; + EXPECT_EQ(ParseArgs(parsers, 5, const_cast<char**>(argv6)), OkStatus()); + CheckArgs(parsers, true, 300, 33333); + + char const* argv7[] = {"test-bin", "-r", "400", "--verbose"}; + EXPECT_EQ(ParseArgs(parsers, 4, const_cast<char**>(argv7)), OkStatus()); + CheckArgs(parsers, true, 400, 11111); + + char const* argv8[] = {"test-bin", "--no-verbose", "-r", "5000", "55555"}; + EXPECT_EQ(ParseArgs(parsers, 5, const_cast<char**>(argv8)), OkStatus()); + CheckArgs(parsers, false, 5000, 55555); +} + +} // namespace +} // namespace pw::rpc::fuzz diff --git a/pw_rpc/fuzz/client_fuzzer.cc b/pw_rpc/fuzz/client_fuzzer.cc new file mode 100644 index 000000000..c5086becf --- /dev/null +++ b/pw_rpc/fuzz/client_fuzzer.cc @@ -0,0 +1,111 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// clang-format off +#include "pw_rpc/internal/log_config.h" // PW_LOG_* macros must be first. +// clang-format on + +#include <sys/socket.h> + +#include <cstring> + +#include "pw_log/log.h" +#include "pw_rpc/fuzz/argparse.h" +#include "pw_rpc/fuzz/engine.h" +#include "pw_rpc/integration_testing.h" + +namespace pw::rpc::fuzz { +namespace { + +// This client configures a socket read timeout to allow the RPC dispatch thread +// to exit gracefully. +constexpr timeval kSocketReadTimeout = {.tv_sec = 1, .tv_usec = 0}; + +int FuzzClient(int argc, char** argv) { + // TODO(aarongreen): Incorporate descriptions into usage message. + Vector<ArgParserVariant, 5> parsers{ + // Enables additional logging. + BoolParser("-v", "--verbose").set_default(false), + + // The number of actions to perform as part of the test. A value of 0 runs + // indefinitely. + UnsignedParser<size_t>("-n", "--num-actions").set_default(256), + + // The seed value for the PRNG. A value of 0 generates a seed. + UnsignedParser<uint64_t>("-s", "--seed").set_default(0), + + // The time, in milliseconds, that can elapse without triggering an error. + UnsignedParser<size_t>("-t", "--timeout").set_default(5000), + + // The port use to connect to the `test_rpc_server`. + UnsignedParser<uint16_t>("port").set_default(48000)}; + + if (!ParseArgs(parsers, argc, argv).ok()) { + PrintUsage(parsers, argv[0]); + return 1; + } + + bool verbose; + size_t num_actions; + uint64_t seed; + size_t timeout_ms; + uint16_t port; + if (!GetArg(parsers, "--verbose", &verbose).ok() || + !GetArg(parsers, "--num-actions", &num_actions).ok() || + !GetArg(parsers, "--seed", &seed).ok() || + !GetArg(parsers, "--timeout", &timeout_ms).ok() || + !GetArg(parsers, "port", &port).ok()) { + return 1; + } + + if (!seed) { + seed = chrono::SystemClock::now().time_since_epoch().count(); + } + + if (auto status = integration_test::InitializeClient(port); !status.ok()) { + PW_LOG_ERROR("Failed to initialize client: %s", pw_StatusString(status)); + return 1; + } + + // Set read timout on socket to allow + // pw::rpc::integration_test::TerminateClient() to complete. + int fd = integration_test::GetClientSocketFd(); + if (setsockopt(fd, + SOL_SOCKET, + SO_RCVTIMEO, + &kSocketReadTimeout, + sizeof(kSocketReadTimeout)) != 0) { + PW_LOG_ERROR("Failed to configure socket receive timeout with errno=%d", + errno); + return 1; + } + + if (num_actions == 0) { + num_actions = std::numeric_limits<size_t>::max(); + } + + Fuzzer fuzzer(integration_test::client(), integration_test::kChannelId); + fuzzer.set_verbose(verbose); + fuzzer.set_timeout(std::chrono::milliseconds(timeout_ms)); + fuzzer.Run(seed, num_actions); + integration_test::TerminateClient(); + return 0; +} + +} // namespace +} // namespace pw::rpc::fuzz + +int main(int argc, char** argv) { + return pw::rpc::fuzz::FuzzClient(argc, argv); +} diff --git a/pw_rpc/fuzz/engine.cc b/pw_rpc/fuzz/engine.cc new file mode 100644 index 000000000..b19dfa398 --- /dev/null +++ b/pw_rpc/fuzz/engine.cc @@ -0,0 +1,553 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +// clang-format off +#include "pw_rpc/internal/log_config.h" // PW_LOG_* macros must be first. + +#include "pw_rpc/fuzz/engine.h" +// clang-format on + +#include <algorithm> +#include <cctype> +#include <chrono> +#include <cinttypes> +#include <limits> +#include <mutex> + +#include "pw_assert/check.h" +#include "pw_bytes/span.h" +#include "pw_log/log.h" +#include "pw_span/span.h" +#include "pw_status/status.h" +#include "pw_string/format.h" + +namespace pw::rpc::fuzz { +namespace { + +using namespace std::chrono_literals; + +// Maximum number of bytes written in a single unary or stream request. +constexpr size_t kMaxWriteLen = MaxSafePayloadSize(); +static_assert(kMaxWriteLen * 0x7E <= std::numeric_limits<uint16_t>::max()); + +struct ActiveVisitor final { + using result_type = bool; + result_type operator()(std::monostate&) { return false; } + result_type operator()(pw::rpc::RawUnaryReceiver& call) { + return call.active(); + } + result_type operator()(pw::rpc::RawClientReaderWriter& call) { + return call.active(); + } +}; + +struct CloseClientStreamVisitor final { + using result_type = void; + result_type operator()(std::monostate&) {} + result_type operator()(pw::rpc::RawUnaryReceiver&) {} + result_type operator()(pw::rpc::RawClientReaderWriter& call) { + call.CloseClientStream().IgnoreError(); + } +}; + +struct WriteVisitor final { + using result_type = bool; + result_type operator()(std::monostate&) { return false; } + result_type operator()(pw::rpc::RawUnaryReceiver&) { return false; } + result_type operator()(pw::rpc::RawClientReaderWriter& call) { + if (!call.active()) { + return false; + } + call.Write(data).IgnoreError(); + return true; + } + ConstByteSpan data; +}; + +struct CancelVisitor final { + using result_type = void; + result_type operator()(std::monostate&) {} + result_type operator()(pw::rpc::RawUnaryReceiver& call) { + call.Cancel().IgnoreError(); + } + result_type operator()(pw::rpc::RawClientReaderWriter& call) { + call.Cancel().IgnoreError(); + } +}; + +struct AbandonVisitor final { + using result_type = void; + result_type operator()(std::monostate&) {} + result_type operator()(pw::rpc::RawUnaryReceiver& call) { call.Abandon(); } + result_type operator()(pw::rpc::RawClientReaderWriter& call) { + call.Abandon(); + } +}; + +} // namespace + +// `Action` methods. + +Action::Action(uint32_t encoded) { + // The first byte is used to determine the operation. The ranges used set the + // relative likelihood of each result, e.g. `kWait` is more likely than + // `kAbandon`. + uint32_t raw = encoded & 0xFF; + if (raw == 0) { + op = kSkip; + } else if (raw < 0x60) { + op = kWait; + } else if (raw < 0x80) { + op = kWriteUnary; + } else if (raw < 0xA0) { + op = kWriteStream; + } else if (raw < 0xC0) { + op = kCloseClientStream; + } else if (raw < 0xD0) { + op = kCancel; + } else if (raw < 0xE0) { + op = kAbandon; + } else if (raw < 0xF0) { + op = kSwap; + } else { + op = kDestroy; + } + target = ((encoded & 0xFF00) >> 8) % Fuzzer::kMaxConcurrentCalls; + value = encoded >> 16; +} + +Action::Action(Op op_, size_t target_, uint16_t value_) + : op(op_), target(target_), value(value_) {} + +Action::Action(Op op_, size_t target_, char val, size_t len) + : op(op_), target(target_) { + PW_ASSERT(op == kWriteUnary || op == kWriteStream); + value = static_cast<uint16_t>(((val % 0x80) * kMaxWriteLen) + + (len % kMaxWriteLen)); +} + +char Action::DecodeWriteValue(uint16_t value) { + return static_cast<char>((value / kMaxWriteLen) % 0x7F); +} + +size_t Action::DecodeWriteLength(uint16_t value) { + return value % kMaxWriteLen; +} + +uint32_t Action::Encode() const { + uint32_t encoded = 0; + switch (op) { + case kSkip: + encoded = 0x00; + break; + case kWait: + encoded = 0x5F; + break; + case kWriteUnary: + encoded = 0x7F; + break; + case kWriteStream: + encoded = 0x9F; + break; + case kCloseClientStream: + encoded = 0xBF; + break; + case kCancel: + encoded = 0xCF; + break; + case kAbandon: + encoded = 0xDF; + break; + case kSwap: + encoded = 0xEF; + break; + case kDestroy: + encoded = 0xFF; + break; + } + encoded |= + ((target < Fuzzer::kMaxConcurrentCalls ? target + : Fuzzer::kMaxConcurrentCalls) % + 0xFF) + << 8; + encoded |= (static_cast<uint32_t>(value) << 16); + return encoded; +} + +void Action::Log(bool verbose, size_t num_actions, const char* fmt, ...) const { + if (!verbose) { + return; + } + char s1[16]; + auto result = callback_id < Fuzzer::kMaxConcurrentCalls + ? string::Format(s1, "%-3zu", callback_id) + : string::Format(s1, "n/a"); + va_list ap; + va_start(ap, fmt); + char s2[128]; + if (result.ok()) { + result = string::FormatVaList(s2, fmt, ap); + } + va_end(ap); + if (result.ok()) { + PW_LOG_INFO("#%-12zu\tthread: %zu\tcallback for: %s\ttarget call: %zu\t%s", + num_actions, + thread_id, + s1, + target, + s2); + } else { + LogFailure(verbose, num_actions, result.status()); + } +} + +void Action::LogFailure(bool verbose, size_t num_actions, Status status) const { + if (verbose && !status.ok()) { + PW_LOG_INFO("#%-12zu\tthread: %zu\tFailed to log action: %s", + num_actions, + thread_id, + pw_StatusString(status)); + } +} + +// FuzzyCall methods. + +void FuzzyCall::RecordWrite(size_t num, bool append) { + std::lock_guard lock(mutex_); + if (append) { + last_write_ += num; + } else { + last_write_ = num; + } + total_written_ += num; + pending_ = true; +} + +void FuzzyCall::Await() { + std::unique_lock<sync::Mutex> lock(mutex_); + cv_.wait(lock, [this]() PW_NO_LOCK_SAFETY_ANALYSIS { return !pending_; }); +} + +void FuzzyCall::Notify() { + if (pending_.exchange(false)) { + cv_.notify_all(); + } +} + +void FuzzyCall::Swap(FuzzyCall& other) { + if (index_ == other.index_) { + return; + } + // Manually acquire locks in an order based on call IDs to prevent deadlock. + if (index_ < other.index_) { + mutex_.lock(); + other.mutex_.lock(); + } else { + other.mutex_.lock(); + mutex_.lock(); + } + call_.swap(other.call_); + std::swap(id_, other.id_); + pending_ = other.pending_.exchange(pending_); + std::swap(last_write_, other.last_write_); + std::swap(total_written_, other.total_written_); + mutex_.unlock(); + other.mutex_.unlock(); + cv_.notify_all(); + other.cv_.notify_all(); +} + +void FuzzyCall::Reset(Variant call) { + { + std::lock_guard lock(mutex_); + call_ = std::move(call); + } + cv_.notify_all(); +} + +void FuzzyCall::Log() { + if (mutex_.try_lock_for(100ms)) { + PW_LOG_INFO("call %zu:", index_); + PW_LOG_INFO(" active: %s", + std::visit(ActiveVisitor(), call_) ? "true" : "false"); + PW_LOG_INFO(" request pending: %s ", pending_ ? "true" : "false"); + PW_LOG_INFO(" last write: %zu bytes", last_write_); + PW_LOG_INFO(" total written: %zu bytes", total_written_); + mutex_.unlock(); + } else { + PW_LOG_WARN("call %zu: failed to acquire lock", index_); + } +} + +// `Fuzzer` methods. + +#define FUZZ_LOG_VERBOSE(...) \ + if (verbose_) { \ + PW_LOG_INFO(__VA_ARGS__); \ + } + +Fuzzer::Fuzzer(Client& client, uint32_t channel_id) + : client_(client, channel_id), + timer_([this](chrono::SystemClock::time_point) { + PW_LOG_ERROR( + "Workers performed %zu actions before timing out without an " + "update.", + num_actions_.load()); + PW_LOG_INFO("Additional call details:"); + for (auto& call : fuzzy_calls_) { + call.Log(); + } + PW_CRASH("Fuzzer found a fatal error condition: TIMEOUT."); + }) { + for (size_t index = 0; index < kMaxConcurrentCalls; ++index) { + fuzzy_calls_.emplace_back(index); + indices_.push_back(index); + contexts_.push_back(CallbackContext{.id = index, .fuzzer = this}); + } +} + +void Fuzzer::Run(uint64_t seed, size_t num_actions) { + FUZZ_LOG_VERBOSE("Fuzzing RPC client with:"); + FUZZ_LOG_VERBOSE(" num_actions: %zu", num_actions); + FUZZ_LOG_VERBOSE(" seed: %" PRIu64, seed); + num_actions_.store(0); + random::XorShiftStarRng64 rng(seed); + while (true) { + { + size_t actions_done = num_actions_.load(); + if (actions_done >= num_actions) { + FUZZ_LOG_VERBOSE("Fuzzing complete; %zu actions performed.", + actions_done); + break; + } + FUZZ_LOG_VERBOSE("%zu actions remaining.", num_actions - actions_done); + } + FUZZ_LOG_VERBOSE("Generating %zu random actions.", kMaxActions); + pw::Vector<uint32_t, kMaxActions> actions; + for (size_t i = 0; i < kNumThreads; ++i) { + size_t num_actions_for_thread; + rng.GetInt(num_actions_for_thread, kMaxActionsPerThread + 1); + for (size_t j = 0; j < num_actions_for_thread; ++j) { + uint32_t encoded = 0; + while (!encoded) { + rng.GetInt(encoded); + } + actions.push_back(encoded); + } + actions.push_back(0); + } + Run(actions); + } +} + +void Fuzzer::Run(const pw::Vector<uint32_t>& actions) { + FUZZ_LOG_VERBOSE("Starting %zu threads to perform %zu actions:", + kNumThreads - 1, + actions.size()); + FUZZ_LOG_VERBOSE(" timeout: %lldms", timer_.timeout() / 1ms); + auto iter = actions.begin(); + timer_.Restart(); + for (size_t thread_id = 0; thread_id < kNumThreads; ++thread_id) { + pw::Vector<uint32_t, kMaxActionsPerThread> thread_actions; + while (thread_actions.size() < kMaxActionsPerThread && + iter != actions.end()) { + uint32_t encoded = *iter++; + if (!encoded) { + break; + } + thread_actions.push_back(encoded); + } + if (thread_id == 0) { + std::lock_guard lock(mutex_); + callback_actions_ = std::move(thread_actions); + callback_iterator_ = callback_actions_.begin(); + } else { + threads_.emplace_back( + [this, thread_id, actions = std::move(thread_actions)]() { + for (const auto& encoded : actions) { + Action action(encoded); + action.set_thread_id(thread_id); + Perform(action); + } + }); + } + } + for (auto& t : threads_) { + t.join(); + } + for (auto& fuzzy_call : fuzzy_calls_) { + fuzzy_call.Reset(); + } + timer_.Cancel(); +} + +void Fuzzer::Perform(const Action& action) { + FuzzyCall& fuzzy_call = FindCall(action.target); + switch (action.op) { + case Action::kSkip: { + if (action.thread_id == 0) { + action.Log(verbose_, ++num_actions_, "Callback chain completed"); + } + break; + } + case Action::kWait: { + if (action.callback_id == action.target) { + // Don't wait in a callback of the target call. + break; + } + if (fuzzy_call.pending()) { + action.Log(verbose_, ++num_actions_, "Waiting for call."); + fuzzy_call.Await(); + } + break; + } + case Action::kWriteUnary: + case Action::kWriteStream: { + if (action.callback_id == action.target) { + // Don't create a new call from the call's own callback. + break; + } + char buf[kMaxWriteLen]; + char val = Action::DecodeWriteValue(action.value); + size_t len = Action::DecodeWriteLength(action.value); + memset(buf, val, len); + if (verbose_) { + char msg_buf[64]; + span msg(msg_buf); + auto result = string::Format( + msg, + "Writing %s request of ", + action.op == Action::kWriteUnary ? "unary" : "stream"); + if (result.ok()) { + size_t off = result.size(); + result = string::Format( + msg.subspan(off), + isprint(val) ? "['%c'; %zu]." : "['\\x%02x'; %zu].", + val, + len); + } + size_t num_actions = ++num_actions_; + if (result.ok()) { + action.Log(verbose_, num_actions, "%s", msg.data()); + } else if (verbose_) { + action.LogFailure(verbose_, num_actions, result.status()); + } + } + bool append = false; + if (action.op == Action::kWriteUnary) { + // Send a unary request. + fuzzy_call.Reset(client_.UnaryEcho( + as_bytes(span(buf, len)), + /* on completed */ + [context = GetContext(action.target)](ConstByteSpan, Status) { + context->fuzzer->OnCompleted(context->id); + }, + /* on error */ + [context = GetContext(action.target)](Status status) { + context->fuzzer->OnError(context->id, status); + })); + + } else if (fuzzy_call.Visit( + WriteVisitor{.data = as_bytes(span(buf, len))})) { + // Append to an existing stream + append = true; + } else { + // .Open a new stream. + fuzzy_call.Reset(client_.BidirectionalEcho( + /* on next */ + [context = GetContext(action.target)](ConstByteSpan) { + context->fuzzer->OnNext(context->id); + }, + /* on completed */ + [context = GetContext(action.target)](Status) { + context->fuzzer->OnCompleted(context->id); + }, + /* on error */ + [context = GetContext(action.target)](Status status) { + context->fuzzer->OnError(context->id, status); + })); + } + fuzzy_call.RecordWrite(len, append); + break; + } + case Action::kCloseClientStream: + action.Log(verbose_, ++num_actions_, "Closing stream."); + fuzzy_call.Visit(CloseClientStreamVisitor()); + break; + case Action::kCancel: + action.Log(verbose_, ++num_actions_, "Canceling call."); + fuzzy_call.Visit(CancelVisitor()); + break; + case Action::kAbandon: { + action.Log(verbose_, ++num_actions_, "Abandoning call."); + fuzzy_call.Visit(AbandonVisitor()); + break; + } + case Action::kSwap: { + size_t other_target = action.value % kMaxConcurrentCalls; + if (action.callback_id == action.target || + action.callback_id == other_target) { + // Don't move a call from within its own callback. + break; + } + action.Log(verbose_, + ++num_actions_, + "Swapping call with call %zu.", + other_target); + std::lock_guard lock(mutex_); + FuzzyCall& other = FindCallLocked(other_target); + std::swap(indices_[fuzzy_call.id()], indices_[other.id()]); + fuzzy_call.Swap(other); + break; + } + case Action::kDestroy: { + if (action.callback_id == action.target) { + // Don't destroy a call from within its own callback. + break; + } + action.Log(verbose_, ++num_actions_, "Destroying call."); + fuzzy_call.Reset(); + break; + } + default: + break; + } + timer_.Restart(); +} + +void Fuzzer::OnNext(size_t callback_id) { FindCall(callback_id).Notify(); } + +void Fuzzer::OnCompleted(size_t callback_id) { + uint32_t encoded = 0; + { + std::lock_guard lock(mutex_); + if (callback_iterator_ != callback_actions_.end()) { + encoded = *callback_iterator_++; + } + } + Action action(encoded); + action.set_callback_id(callback_id); + Perform(action); + FindCall(callback_id).Notify(); +} + +void Fuzzer::OnError(size_t callback_id, Status status) { + FuzzyCall& call = FindCall(callback_id); + PW_LOG_WARN("Call %zu received an error from the server: %s", + call.id(), + pw_StatusString(status)); + call.Notify(); +} + +} // namespace pw::rpc::fuzz diff --git a/pw_rpc/fuzz/engine_test.cc b/pw_rpc/fuzz/engine_test.cc new file mode 100644 index 000000000..5ac1a49a1 --- /dev/null +++ b/pw_rpc/fuzz/engine_test.cc @@ -0,0 +1,264 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#include "pw_rpc/fuzz/engine.h" + +#include <chrono> + +#include "gtest/gtest.h" +#include "pw_containers/vector.h" +#include "pw_log/log.h" +#include "pw_rpc/benchmark.h" +#include "pw_rpc/internal/client_server_testing_threaded.h" +#include "pw_rpc/internal/fake_channel_output.h" +#include "pw_thread/test_threads.h" + +namespace pw::rpc::fuzz { +namespace { + +using namespace std::literals::chrono_literals; + +// Maximum time, in milliseconds, that can elapse without a call completing or +// being dropped in some way.. +const chrono::SystemClock::duration kTimeout = 5s; + +// These are fairly tight constraints in order to fit within the default +// `PW_UNIT_TEST_CONFIG_MEMORY_POOL_SIZE`. +constexpr size_t kMaxPackets = 128; +constexpr size_t kMaxPayloadSize = 64; + +using BufferedChannelOutputBase = + internal::test::FakeChannelOutputBuffer<kMaxPackets, kMaxPayloadSize>; + +/// Channel output backed by a fixed buffer. +class BufferedChannelOutput : public BufferedChannelOutputBase { + public: + BufferedChannelOutput() : BufferedChannelOutputBase() {} +}; + +using FuzzerChannelOutputBase = + internal::WatchableChannelOutput<BufferedChannelOutput, + kMaxPayloadSize, + kMaxPackets, + kMaxPayloadSize>; + +/// Channel output that can be waited on by the server. +class FuzzerChannelOutput : public FuzzerChannelOutputBase { + public: + FuzzerChannelOutput() : FuzzerChannelOutputBase() {} +}; + +using FuzzerContextBase = + internal::ClientServerTestContextThreaded<FuzzerChannelOutput, + kMaxPayloadSize, + kMaxPackets, + kMaxPayloadSize>; +class FuzzerContext : public FuzzerContextBase { + public: + static constexpr uint32_t kChannelId = 1; + + FuzzerContext() : FuzzerContextBase(thread::test::TestOptionsThread0()) {} +}; + +class RpcFuzzTestingTest : public testing::Test { + protected: + void SetUp() override { context_.server().RegisterService(service_); } + + void Add(Action::Op op, size_t target, uint16_t value) { + actions_.push_back(Action(op, target, value).Encode()); + } + + void Add(Action::Op op, size_t target, char val, size_t len) { + actions_.push_back(Action(op, target, val, len).Encode()); + } + + void NextThread() { actions_.push_back(0); } + + void Run() { + Fuzzer fuzzer(context_.client(), FuzzerContext::kChannelId); + fuzzer.set_verbose(true); + fuzzer.set_timeout(kTimeout); + fuzzer.Run(actions_); + } + + private: + FuzzerContext context_; + BenchmarkService service_; + Vector<uint32_t, Fuzzer::kMaxActions> actions_; +}; + +TEST_F(RpcFuzzTestingTest, SequentialRequests) { + // Callback thread + Add(Action::kWriteStream, 1, 'B', 1); + Add(Action::kSkip, 0, 0); + Add(Action::kWriteStream, 2, 'B', 2); + Add(Action::kSkip, 0, 0); + Add(Action::kWriteStream, 3, 'B', 3); + Add(Action::kSkip, 0, 0); + NextThread(); + + // Thread 1 + Add(Action::kWriteStream, 0, 'A', 2); + Add(Action::kWait, 1, 0); + Add(Action::kWriteStream, 1, 'A', 4); + NextThread(); + + // Thread 2 + NextThread(); + Add(Action::kWait, 2, 0); + Add(Action::kWriteStream, 2, 'A', 6); + + // Thread 3 + NextThread(); + Add(Action::kWait, 3, 0); + + Run(); +} + +// TODO(b/274437709): Re-enable. +TEST_F(RpcFuzzTestingTest, DISABLED_SimultaneousRequests) { + // Callback thread + NextThread(); + + // Thread 1 + Add(Action::kWriteUnary, 1, 'A', 1); + Add(Action::kWait, 2, 0); + NextThread(); + + // Thread 2 + Add(Action::kWriteUnary, 2, 'B', 2); + Add(Action::kWait, 3, 0); + NextThread(); + + // Thread 3 + Add(Action::kWriteUnary, 3, 'C', 3); + Add(Action::kWait, 1, 0); + NextThread(); + + Run(); +} + +// TODO(b/274437709) This test currently does not pass as it exhausts the fake +// channel. It will be re-enabled when the underlying stream is swapped for +// a pw_ring_buffer-based approach. +TEST_F(RpcFuzzTestingTest, DISABLED_CanceledRequests) { + // Callback thread + NextThread(); + + // Thread 1 + for (size_t i = 0; i < 10; ++i) { + Add(Action::kWriteUnary, i % 3, 'A', i); + } + Add(Action::kWait, 0, 0); + Add(Action::kWait, 1, 0); + Add(Action::kWait, 2, 0); + NextThread(); + + // Thread 2 + for (size_t i = 0; i < 10; ++i) { + Add(Action::kCancel, i % 3, 0); + } + NextThread(); + + // Thread 3 + NextThread(); + + Run(); +} + +// TODO(b/274437709) This test currently does not pass as it exhausts the fake +// channel. It will be re-enabled when the underlying stream is swapped for +// a pw_ring_buffer-based approach. +TEST_F(RpcFuzzTestingTest, DISABLED_AbandonedRequests) { + // Callback thread + NextThread(); + + // Thread 1 + for (size_t i = 0; i < 10; ++i) { + Add(Action::kWriteUnary, i % 3, 'A', i); + } + Add(Action::kWait, 0, 0); + Add(Action::kWait, 1, 0); + Add(Action::kWait, 2, 0); + NextThread(); + + // Thread 2 + for (size_t i = 0; i < 10; ++i) { + Add(Action::kAbandon, i % 3, 0); + } + NextThread(); + + // Thread 3 + NextThread(); + + Run(); +} + +// TODO(b/274437709) This test currently does not pass as it exhausts the fake +// channel. It will be re-enabled when the underlying stream is swapped for +// a pw_ring_buffer-based approach. +TEST_F(RpcFuzzTestingTest, DISABLED_SwappedRequests) { + Vector<uint32_t, Fuzzer::kMaxActions> actions; + // Callback thread + NextThread(); + // Thread 1 + for (size_t i = 0; i < 10; ++i) { + Add(Action::kWriteUnary, i % 3, 'A', i); + } + Add(Action::kWait, 0, 0); + Add(Action::kWait, 1, 0); + Add(Action::kWait, 2, 0); + NextThread(); + // Thread 2 + for (size_t i = 0; i < 100; ++i) { + auto j = i % 3; + Add(Action::kSwap, j, j + 1); + } + NextThread(); + // Thread 3 + NextThread(); + + Run(); +} + +// TODO(b/274437709) This test currently does not pass as it exhausts the fake +// channel. It will be re-enabled when the underlying stream is swapped for +// a pw_ring_buffer-based approach. +TEST_F(RpcFuzzTestingTest, DISABLED_DestroyedRequests) { + // Callback thread + NextThread(); + + // Thread 1 + for (size_t i = 0; i < 100; ++i) { + Add(Action::kWriteUnary, i % 3, 'A', i); + } + Add(Action::kWait, 0, 0); + Add(Action::kWait, 1, 0); + Add(Action::kWait, 2, 0); + NextThread(); + + // Thread 2 + for (size_t i = 0; i < 100; ++i) { + Add(Action::kDestroy, i % 3, 0); + } + NextThread(); + + // Thread 3 + NextThread(); + + Run(); +} + +} // namespace +} // namespace pw::rpc::fuzz diff --git a/pw_rpc/fuzz/public/pw_rpc/fuzz/alarm_timer.h b/pw_rpc/fuzz/public/pw_rpc/fuzz/alarm_timer.h new file mode 100644 index 000000000..9ccd7ce0b --- /dev/null +++ b/pw_rpc/fuzz/public/pw_rpc/fuzz/alarm_timer.h @@ -0,0 +1,56 @@ +// Copyright 2022 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#pragma once + +#include "pw_chrono/system_clock.h" +#include "pw_chrono/system_timer.h" + +namespace pw::rpc::fuzz { + +/// Represents a timer that invokes a callback on timeout. Once started, it will +/// invoke the callback after a provided duration unless it is restarted, +/// canceled, or destroyed. +class AlarmTimer { + public: + AlarmTimer(chrono::SystemTimer::ExpiryCallback&& on_timeout) + : timer_(std::move(on_timeout)) {} + + chrono::SystemClock::duration timeout() const { return timeout_; } + + /// "Arms" the timer. The callback will be invoked if `timeout` elapses + /// without a call to `Restart`, `Cancel`, or the destructor. Calling `Start` + /// again restarts the timer, possibly with a different `timeout` value. + void Start(chrono::SystemClock::duration timeout) { + timeout_ = timeout; + Restart(); + } + + /// Restarts the timer. This is equivalent to calling `Start` with the same + /// `timeout` as passed previously. Does nothing if `Start` has not been + /// called. + void Restart() { + Cancel(); + timer_.InvokeAfter(timeout_); + } + + /// "Disarms" the timer. The callback will not be invoked unless `Start` is + /// called again. Does nothing if `Start` has not been called. + void Cancel() { timer_.Cancel(); } + + private: + chrono::SystemTimer timer_; + chrono::SystemClock::duration timeout_; +}; + +} // namespace pw::rpc::fuzz diff --git a/pw_rpc/fuzz/public/pw_rpc/fuzz/argparse.h b/pw_rpc/fuzz/public/pw_rpc/fuzz/argparse.h new file mode 100644 index 000000000..05a7633e6 --- /dev/null +++ b/pw_rpc/fuzz/public/pw_rpc/fuzz/argparse.h @@ -0,0 +1,230 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#pragma once + +/// Command line argument parsing. +/// +/// The objects defined below can be used to parse command line arguments of +/// different types. These objects are "just enough" defined for current use +/// cases, but the design is intended to be extensible as new types and traits +/// are needed. +/// +/// Example: +/// +/// Given a boolean flag "verbose", a numerical flag "runs", and a positional +/// "port" argument to be parsed, we can create a vector of parsers. In this +/// example, we modify the parsers during creation to set default values: +/// +/// @code +/// Vector<ArgParserVariant, 3> parsers = { +/// BoolParser("-v", "--verbose").set_default(false), +/// UnsignedParser<size_t>("-r", "--runs").set_default(1000), +/// UnsignedParser<uint16_t>("port").set_default(11111), +/// }; +/// @endcode +/// +/// With this vector, we can then parse command line arguments and extract +/// the values of arguments that were set, e.g.: +/// +/// @code +/// if (!ParseArgs(parsers, argc, argv).ok()) { +/// PrintUsage(parsers, argv[0]); +/// return 1; +/// } +/// bool verbose; +/// size_t runs; +/// uint16_t port; +/// if (!GetArg(parsers, "--verbose", &verbose).ok() || +/// !GetArg(parsers, "--runs", &runs).ok() || +/// !GetArg(parsers, "port", &port).ok()) { +/// // Shouldn't happen unless names do not match. +/// return 1; +/// } +/// +/// // Do stuff with `verbose`, `runs`, and `port`... +/// @endcode + +#include <cstddef> +#include <cstdint> +#include <string_view> +#include <variant> + +#include "pw_containers/vector.h" +#include "pw_status/status.h" + +namespace pw::rpc::fuzz { + +/// Enumerates the results of trying to parse a specific command line argument +/// with a particular parsers. +enum ParseStatus { + /// The argument matched the parser and was successfully parsed without a + /// value. + kParsedOne, + + /// The argument matched the parser and was successfully parsed with a value. + kParsedTwo, + + /// The argument did not match the parser. This is not necessarily an error; + /// the argument may match a different parser. + kParseMismatch, + + /// The argument matched a parser, but could not be parsed. This may be due to + /// a missing value for a flag, a value of the wrong type, a provided value + /// being out of range, etc. Parsers should log additional details before + /// returning this value. + kParseFailure, +}; + +/// Holds parsed argument values of different types. +using ArgVariant = std::variant<std::monostate, bool, uint64_t>; + +/// Base class for argument parsers. +class ArgParserBase { + public: + virtual ~ArgParserBase() = default; + + std::string_view short_name() const { return short_name_; } + std::string_view long_name() const { return long_name_; } + bool positional() const { return positional_; } + + /// Clears the value. Typically, command line arguments are only parsed once, + /// but this method is useful for testing. + void Reset() { value_ = std::monostate(); } + + protected: + /// Defines an argument parser with a single name. This may be a positional + /// argument or a flag. + ArgParserBase(std::string_view name); + + /// Defines an argument parser for a flag with short and long names. + ArgParserBase(std::string_view shortopt, std::string_view longopt); + + void set_initial(ArgVariant initial) { initial_ = initial; } + void set_value(ArgVariant value) { value_ = value; } + + /// Examines if the given `arg` matches this parser. A parser for a flag can + /// match the short name (e.g. '-f') if set, or the long name (e.g. '--foo'). + /// A parser for a positional argument will match anything until it has a + /// value set. + bool Match(std::string_view arg); + + /// Returns the parsed value. + template <typename T> + T Get() const { + return std::get<T>(GetValue()); + } + + private: + const ArgVariant& GetValue() const; + + std::string_view short_name_; + std::string_view long_name_; + bool positional_; + + ArgVariant initial_; + ArgVariant value_; +}; + +// Argument parsers for boolean arguments. These arguments are always flags, and +// can be specified as, e.g. "-f" (true), "--foo" (true) or "--no-foo" (false). +class BoolParser : public ArgParserBase { + public: + BoolParser(std::string_view optname); + BoolParser(std::string_view shortopt, std::string_view longopt); + + bool value() const { return Get<bool>(); } + BoolParser& set_default(bool value); + + ParseStatus Parse(std::string_view arg0, + std::string_view arg1 = std::string_view()); +}; + +// Type-erasing argument parser for unsigned integer arguments. This object +// always parses values as `uint64_t`s and should not be used directly. +// Instead, use `UnsignedParser<T>` with a type to explicitly narrow to. +class UnsignedParserBase : public ArgParserBase { + protected: + UnsignedParserBase(std::string_view name); + UnsignedParserBase(std::string_view shortopt, std::string_view longopt); + + ParseStatus Parse(std::string_view arg0, std::string_view arg1, uint64_t max); +}; + +// Argument parser for unsigned integer arguments. These arguments may be flags +// or positional arguments. +template <typename T, typename std::enable_if_t<std::is_unsigned_v<T>, int> = 0> +class UnsignedParser : public UnsignedParserBase { + public: + UnsignedParser(std::string_view name) : UnsignedParserBase(name) {} + UnsignedParser(std::string_view shortopt, std::string_view longopt) + : UnsignedParserBase(shortopt, longopt) {} + + T value() const { return static_cast<T>(Get<uint64_t>()); } + + UnsignedParser& set_default(T value) { + set_initial(static_cast<uint64_t>(value)); + return *this; + } + + ParseStatus Parse(std::string_view arg0, + std::string_view arg1 = std::string_view()) { + return UnsignedParserBase::Parse(arg0, arg1, std::numeric_limits<T>::max()); + } +}; + +// Holds argument parsers of different types. +using ArgParserVariant = + std::variant<BoolParser, UnsignedParser<uint16_t>, UnsignedParser<size_t>>; + +// Parses the command line arguments and sets the values of the given `parsers`. +Status ParseArgs(Vector<ArgParserVariant>& parsers, int argc, char** argv); + +// Logs a usage message based on the given `parsers` and the program name given +// by `argv0`. +void PrintUsage(const Vector<ArgParserVariant>& parsers, + std::string_view argv0); + +// Attempts to find the parser in `parsers` with the given `name`, and returns +// its value if found. +std::optional<ArgVariant> GetArg(const Vector<ArgParserVariant>& parsers, + std::string_view name); + +inline void GetArgValue(const ArgVariant& arg, bool* out) { + *out = std::get<bool>(arg); +} + +template <typename T, typename std::enable_if_t<std::is_unsigned_v<T>, int> = 0> +void GetArgValue(const ArgVariant& arg, T* out) { + *out = static_cast<T>(std::get<uint64_t>(arg)); +} + +// Like `GetArgVariant` above, but extracts the typed value from the variant +// into `out`. Returns an error if no parser exists in `parsers` with the given +// `name`. +template <typename T> +Status GetArg(const Vector<ArgParserVariant>& parsers, + std::string_view name, + T* out) { + const auto& arg = GetArg(parsers, name); + if (!arg.has_value()) { + return Status::InvalidArgument(); + } + GetArgValue(*arg, out); + return OkStatus(); +} + +// Resets the parser with the given name. Returns an error if not found. +Status ResetArg(Vector<ArgParserVariant>& parsers, std::string_view name); + +} // namespace pw::rpc::fuzz diff --git a/pw_rpc/fuzz/public/pw_rpc/fuzz/engine.h b/pw_rpc/fuzz/public/pw_rpc/fuzz/engine.h new file mode 100644 index 000000000..34e92c003 --- /dev/null +++ b/pw_rpc/fuzz/public/pw_rpc/fuzz/engine.h @@ -0,0 +1,339 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#pragma once + +#include <atomic> +#include <cstdarg> +#include <cstddef> +#include <cstdint> +#include <thread> +#include <variant> + +#include "pw_containers/vector.h" +#include "pw_random/xor_shift.h" +#include "pw_rpc/benchmark.h" +#include "pw_rpc/benchmark.raw_rpc.pb.h" +#include "pw_rpc/fuzz/alarm_timer.h" +#include "pw_sync/condition_variable.h" +#include "pw_sync/lock_annotations.h" +#include "pw_sync/mutex.h" +#include "pw_sync/timed_mutex.h" + +namespace pw::rpc::fuzz { + +/// Describes an action a fuzzing thread can perform on a call. +struct Action { + enum Op : uint8_t { + /// No-op. + kSkip, + + /// Waits for the call indicated by `target` to complete. + kWait, + + /// Makes a new unary request using the call indicated by `target`. The data + /// written is derived from `value`. + kWriteUnary, + + /// Writes to a stream request using the call indicated by `target`, or + /// makes + /// a new one if not currently a stream call. The data written is derived + /// from `value`. + kWriteStream, + + /// Closes the stream if the call indicated by `target` is a stream call. + kCloseClientStream, + + /// Cancels the call indicated by `target`. + kCancel, + + /// Abandons the call indicated by `target`. + kAbandon, + + /// Swaps the call indicated by `target` with a call indicated by `value`. + kSwap, + + /// Sets the call indicated by `target` to an initial, unset state. + kDestroy, + }; + + constexpr Action() = default; + Action(uint32_t encoded); + Action(Op op, size_t target, uint16_t value); + Action(Op op, size_t target, char val, size_t len); + ~Action() = default; + + void set_thread_id(size_t thread_id_) { + thread_id = thread_id_; + callback_id = std::numeric_limits<size_t>::max(); + } + + void set_callback_id(size_t callback_id_) { + thread_id = 0; + callback_id = callback_id_; + } + + // For a write action's value, returns the character value to be written. + static char DecodeWriteValue(uint16_t value); + + // For a write action's value, returns the number of characters to be written. + static size_t DecodeWriteLength(uint16_t value); + + /// Returns a value that represents the fields of an action. Constructing an + /// `Action` with this value will produce the same fields. + uint32_t Encode() const; + + /// Records details of the action being performed if verbose logging is + /// enabled. + void Log(bool verbose, size_t num_actions, const char* fmt, ...) const; + + /// Records an encountered when trying to log an action. + void LogFailure(bool verbose, size_t num_actions, Status status) const; + + Op op = kSkip; + size_t target = 0; + uint16_t value = 0; + + size_t thread_id = 0; + size_t callback_id = std::numeric_limits<size_t>::max(); +}; + +/// Wraps an RPC call that may be either a `RawUnaryReceiver` or +/// `RawClientReaderWriter`. Allows applying `Action`s to each possible +/// type of call. +class FuzzyCall { + public: + using Variant = + std::variant<std::monostate, RawUnaryReceiver, RawClientReaderWriter>; + + explicit FuzzyCall(size_t index) : index_(index), id_(index) {} + ~FuzzyCall() = default; + + size_t id() { + std::lock_guard lock(mutex_); + return id_; + } + + bool pending() { + std::lock_guard lock(mutex_); + return pending_; + } + + /// Applies the given visitor to the call variant. If the action taken by the + /// visitor is expected to complete the call, it will notify any threads + /// waiting for the call to complete. This version of the method does not + /// return the result of the visiting the variant. + template <typename Visitor, + typename std::enable_if_t< + std::is_same_v<typename Visitor::result_type, void>, + int> = 0> + typename Visitor::result_type Visit(Visitor visitor, bool completes = true) { + { + std::lock_guard lock(mutex_); + std::visit(std::move(visitor), call_); + } + if (completes && pending_.exchange(false)) { + cv_.notify_all(); + } + } + + /// Applies the given visitor to the call variant. If the action taken by the + /// visitor is expected to complete the call, it will notify any threads + /// waiting for the call to complete. This version of the method returns the + /// result of the visiting the variant. + template <typename Visitor, + typename std::enable_if_t< + !std::is_same_v<typename Visitor::result_type, void>, + int> = 0> + typename Visitor::result_type Visit(Visitor visitor, bool completes = true) { + typename Visitor::result_type result; + { + std::lock_guard lock(mutex_); + result = std::visit(std::move(visitor), call_); + } + if (completes && pending_.exchange(false)) { + cv_.notify_all(); + } + return result; + } + + // Records the number of bytes written as part of a request. If `append` is + // true, treats the write as a continuation of a streaming request. + void RecordWrite(size_t num, bool append = false); + + /// Waits to be notified that a callback has been invoked. + void Await() PW_LOCKS_EXCLUDED(mutex_); + + /// Completes the call, notifying any waiters. + void Notify() PW_LOCKS_EXCLUDED(mutex_); + + /// Exchanges the call represented by this object with another. + void Swap(FuzzyCall& other); + + /// Resets the call wrapped by this object with a new one. Destorys the + /// previous call. + void Reset(Variant call = Variant()) PW_LOCKS_EXCLUDED(mutex_); + + // Reports the state of this object. + void Log() PW_LOCKS_EXCLUDED(mutex_); + + private: + /// This represents the index in the engine's list of calls. It is used to + /// ensure a consistent order of locking multiple calls. + const size_t index_; + + sync::TimedMutex mutex_; + sync::ConditionVariable cv_; + + /// An identifier that can be used find this object, e.g. by a callback, even + /// when it has been swapped with another call. + size_t id_ PW_GUARDED_BY(mutex_); + + /// Holds the actual pw::rpc::Call object, when present. + Variant call_ PW_GUARDED_BY(mutex_); + + /// Set when a request is sent, and cleared when a callback is invoked. + std::atomic_bool pending_ = false; + + /// Bytes sent in the last unary request or stream write. + size_t last_write_ PW_GUARDED_BY(mutex_) = 0; + + /// Total bytes sent using this call object. + size_t total_written_ PW_GUARDED_BY(mutex_) = 0; +}; + +/// The main RPC fuzzing engine. +/// +/// This class takes or generates a sequence of actions, and dsitributes them to +/// a number of threads that can perform them using an RPC client. Passing the +/// same seed to the engine at construction will allow it to generate the same +/// sequence of actions. +class Fuzzer { + public: + /// Number of fuzzing threads. The first thread counted is the RPC dispatch + /// thread. + static constexpr size_t kNumThreads = 4; + + /// Maximum number of actions that a single thread will try to perform before + /// exiting. + static constexpr size_t kMaxActionsPerThread = 255; + + /// The number of call objects available to be used for fuzzing. + static constexpr size_t kMaxConcurrentCalls = 8; + + /// The mxiumum number of individual fuzzing actions that the fuzzing threads + /// can perform. The `+ 1` is to allow the inclusion of a special `0` action + /// to separate each thread's actions when concatenated into a single list. + static constexpr size_t kMaxActions = + kNumThreads * (kMaxActionsPerThread + 1); + + explicit Fuzzer(Client& client, uint32_t channel_id); + + /// The fuzzer engine should remain pinned in memory since it is referenced by + /// the `CallbackContext`s. + Fuzzer(const Fuzzer&) = delete; + Fuzzer(Fuzzer&&) = delete; + Fuzzer& operator=(const Fuzzer&) = delete; + Fuzzer& operator=(Fuzzer&&) = delete; + + void set_verbose(bool verbose) { verbose_ = verbose; } + + /// Sets the timeout and starts the timer. + void set_timeout(chrono::SystemClock::duration timeout) { + timer_.Start(timeout); + } + + /// Generates encoded actions from the RNG and `Run`s them. + void Run(uint64_t seed, size_t num_actions); + + /// Splits the provided `actions` between the fuzzing threads and runs them to + /// completion. + void Run(const Vector<uint32_t>& actions); + + private: + /// Information passed to the RPC callbacks, including the index of the + /// associated call and a pointer to the fuzzer object. + struct CallbackContext { + size_t id; + Fuzzer* fuzzer; + }; + + /// Restarts the alarm timer, delaying it from detecting a timeout. This is + /// called whenever actions complete and indicates progress is still being + /// made. + void ResetTimerLocked() PW_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + /// Decodes the `encoded` action and performs it. The `thread_id` is used for + /// verbose diagnostics. When invoked from `PerformCallback` the `callback_id` + /// will be set to the index of the associated call. This allows avoiding + /// specific, prohibited actions, e.g. destroying a call from its own + /// callback. + void Perform(const Action& action) PW_LOCKS_EXCLUDED(mutex_); + + /// Returns the call with the matching `id`. + FuzzyCall& FindCall(size_t id) PW_LOCKS_EXCLUDED(mutex_) { + std::lock_guard lock(mutex_); + return FindCallLocked(id); + } + + FuzzyCall& FindCallLocked(size_t id) PW_EXCLUSIVE_LOCKS_REQUIRED(mutex_) { + return fuzzy_calls_[indices_[id]]; + } + + /// Returns a pointer to callback context for the given call index. + CallbackContext* GetContext(size_t callback_id) PW_LOCKS_EXCLUDED(mutex_) { + std::lock_guard lock(mutex_); + return &contexts_[callback_id]; + } + + /// Callback for stream write made by the call with the given `callback_id`. + void OnNext(size_t callback_id) PW_LOCKS_EXCLUDED(mutex_); + + /// Callback for completed request for the call with the given `callback_id`. + void OnCompleted(size_t callback_id) PW_LOCKS_EXCLUDED(mutex_); + + /// Callback for an error for the call with the given `callback_id`. + void OnError(size_t callback_id, Status status) PW_LOCKS_EXCLUDED(mutex_); + + bool verbose_ = false; + pw_rpc::raw::Benchmark::Client client_; + BenchmarkService service_; + + /// Alarm thread that detects when no workers have made recent progress. + AlarmTimer timer_; + + sync::Mutex mutex_; + + /// Worker threads. The first thread is the RPC response dispatcher. + Vector<std::thread, kNumThreads> threads_; + + /// RPC call objects. + Vector<FuzzyCall, kMaxConcurrentCalls> fuzzy_calls_; + + /// Maps each call's IDs to its index. Since calls may be move before their + /// callbacks are invoked, this list can be used to find the original call. + Vector<size_t, kMaxConcurrentCalls> indices_ PW_GUARDED_BY(mutex_); + + /// Context objects used to reference the engine and call. + Vector<CallbackContext, kMaxConcurrentCalls> contexts_ PW_GUARDED_BY(mutex_); + + /// Set of actions performed as callbacks from other calls. + Vector<uint32_t, kMaxActionsPerThread> callback_actions_ + PW_GUARDED_BY(mutex_); + Vector<uint32_t>::iterator callback_iterator_ PW_GUARDED_BY(mutex_); + + /// Total actions performed by all workers. + std::atomic<size_t> num_actions_ = 0; +}; + +} // namespace pw::rpc::fuzz diff --git a/pw_rpc/internal/integration_test_ports.gni b/pw_rpc/internal/integration_test_ports.gni index 5413f8796..5197aab46 100644 --- a/pw_rpc/internal/integration_test_ports.gni +++ b/pw_rpc/internal/integration_test_ports.gni @@ -16,4 +16,5 @@ # in one place to prevent accidental conflicts between tests. pw_rpc_PYTHON_CLIENT_CPP_SERVER_TEST_PORT = 30576 pw_rpc_CPP_CLIENT_INTEGRATION_TEST_PORT = 30577 +pw_rpc_CPP_CLIENT_FUZZER_TEST_PORT = 30578 pw_unit_test_RPC_SERVICE_TEST_PORT = 30580 diff --git a/pw_rpc/internal/packet.proto b/pw_rpc/internal/packet.proto index eb29072c6..606efb9ef 100644 --- a/pw_rpc/internal/packet.proto +++ b/pw_rpc/internal/packet.proto @@ -33,10 +33,6 @@ enum PacketType { // The client received a packet for an RPC it did not request. CLIENT_ERROR = 4; - // Deprecated, do not use. Send a CLIENT_ERROR with status CANCELLED instead. - // TODO(b/234879973): Remove this packet type. - DEPRECATED_CANCEL = 6; - // A client stream has completed. CLIENT_STREAM_END = 8; @@ -45,16 +41,15 @@ enum PacketType { // The RPC has finished. RESPONSE = 1; - // Deprecated, do not use. Formerly was used as the last packet in a server - // stream. - // TODO(b/234879973): Remove this packet type. - DEPRECATED_SERVER_STREAM_END = 3; - // The server was unable to process a request. SERVER_ERROR = 5; // A message in a server stream. SERVER_STREAM = 7; + + // Reserve field numbers for deprecated PacketTypes. + reserved 3; // SERVER_STREAM_END (equivalent to RESPONSE now) + reserved 6; // CANCEL (replaced by CLIENT_ERROR with status CANCELLED) } message RpcPacket { diff --git a/pw_rpc/packet.cc b/pw_rpc/packet.cc index 2b9285071..3e9803c23 100644 --- a/pw_rpc/packet.cc +++ b/pw_rpc/packet.cc @@ -79,12 +79,6 @@ Result<Packet> Packet::FromBuffer(ConstByteSpan data) { return status; } - // TODO(b/234879973): CANCEL is equivalent to CLIENT_ERROR with status - // CANCELLED. Remove this workaround when CANCEL is removed. - if (packet.type() == PacketType::DEPRECATED_CANCEL) { - packet.set_status(Status::Cancelled()); - } - return packet; } diff --git a/pw_rpc/public/pw_rpc/internal/config.h b/pw_rpc/public/pw_rpc/internal/config.h index 45fc79918..24a227159 100644 --- a/pw_rpc/public/pw_rpc/internal/config.h +++ b/pw_rpc/public/pw_rpc/internal/config.h @@ -24,7 +24,7 @@ /// /// This option controls whether or not include a callback that is called when /// the client stream ends. The callback is included in all ServerReader/Writer -/// objects as a @cpp_class{pw::Function}, so may have a significant cost. +/// objects as a @cpp_type{pw::Function}, so may have a significant cost. /// /// This is disabled by default. #ifndef PW_RPC_CLIENT_STREAM_END_CALLBACK @@ -160,13 +160,16 @@ static_assert( /// If @c_macro{PW_RPC_DYNAMIC_ALLOCATION} is enabled, this macro must expand to /// a container capable of storing objects of the provided type. This container -/// will be used internally be pw_rpc. Defaults to `std::vector<type>`, but may -/// be set to any type that supports the following std::vector operations: +/// will be used internally by pw_rpc to allocate the channels list and encoding +/// buffer. Defaults to `std::vector<type>`, but may be set to any type that +/// supports the following `std::vector` operations: /// /// - Default construction /// - `emplace_back()` /// - `pop_back()` /// - `back()` +/// - `resize()` +/// - `clear()` /// - Range-based for loop iteration (`begin()`, `end()`) /// #ifndef PW_RPC_DYNAMIC_CONTAINER diff --git a/pw_rpc/public/pw_rpc/internal/encoding_buffer.h b/pw_rpc/public/pw_rpc/internal/encoding_buffer.h index c3b8033e6..8ae752f12 100644 --- a/pw_rpc/public/pw_rpc/internal/encoding_buffer.h +++ b/pw_rpc/public/pw_rpc/internal/encoding_buffer.h @@ -21,20 +21,47 @@ #include "pw_assert/assert.h" #include "pw_bytes/span.h" +#include "pw_rpc/internal/config.h" #include "pw_rpc/internal/lock.h" #include "pw_rpc/internal/packet.h" #include "pw_status/status_with_size.h" +#if PW_RPC_DYNAMIC_ALLOCATION + +#include PW_RPC_DYNAMIC_CONTAINER_INCLUDE + +#endif // PW_RPC_DYNAMIC_ALLOCATION + namespace pw::rpc::internal { constexpr ByteSpan ResizeForPayload(ByteSpan buffer) { return buffer.subspan(Packet::kMinEncodedSizeWithoutPayload); } +// Wraps a statically allocated encoding buffer. +class StaticEncodingBuffer { + public: + constexpr StaticEncodingBuffer() : buffer_{} {} + + ByteSpan AllocatePayloadBuffer() { return ResizeForPayload(buffer_); } + ByteSpan GetPacketBuffer(size_t /* payload_size */) { return buffer_; } + + void Release() {} + void ReleaseIfAllocated() {} + + private: + static_assert(MaxSafePayloadSize() > 0, + "pw_rpc's encode buffer is too small to fit any data"); + + std::array<std::byte, cfg::kEncodingBufferSizeBytes> buffer_; +}; + +#if PW_RPC_DYNAMIC_ALLOCATION + // Wraps a dynamically allocated encoding buffer. class DynamicEncodingBuffer { public: - constexpr DynamicEncodingBuffer() = default; + DynamicEncodingBuffer() = default; ~DynamicEncodingBuffer() { PW_DASSERT(buffer_.empty()); } @@ -56,8 +83,7 @@ class DynamicEncodingBuffer { // Frees the payload buffer, which MUST have been allocated previously. void Release() { PW_DASSERT(!buffer_.empty()); - delete[] buffer_.data(); - buffer_ = {}; + buffer_.clear(); } // Frees the payload buffer, if one was allocated. @@ -72,37 +98,23 @@ class DynamicEncodingBuffer { const size_t buffer_size = payload_size + Packet::kMinEncodedSizeWithoutPayload; PW_DASSERT(buffer_.empty()); - buffer_ = span(new std::byte[buffer_size], buffer_size); + buffer_.resize(buffer_size); } - ByteSpan buffer_; + PW_RPC_DYNAMIC_CONTAINER(std::byte) buffer_; }; -// Wraps a statically allocated encoding buffer. -class StaticEncodingBuffer { - public: - constexpr StaticEncodingBuffer() : buffer_{} {} +using EncodingBuffer = DynamicEncodingBuffer; - ByteSpan AllocatePayloadBuffer() { return ResizeForPayload(buffer_); } - ByteSpan GetPacketBuffer(size_t /* payload_size */) { return buffer_; } - - void Release() {} - void ReleaseIfAllocated() {} +#else - private: - static_assert(MaxSafePayloadSize() > 0, - "pw_rpc's encode buffer is too small to fit any data"); +using EncodingBuffer = StaticEncodingBuffer; - std::array<std::byte, cfg::kEncodingBufferSizeBytes> buffer_; -}; +#endif // PW_RPC_DYNAMIC_ALLOCATION // Instantiate the global encoding buffer variable, depending on whether dynamic // allocation is enabled or not. -#if PW_RPC_DYNAMIC_ALLOCATION -inline DynamicEncodingBuffer encoding_buffer PW_GUARDED_BY(rpc_lock()); -#else -inline StaticEncodingBuffer encoding_buffer PW_GUARDED_BY(rpc_lock()); -#endif // PW_RPC_DYNAMIC_ALLOCATION +inline EncodingBuffer encoding_buffer PW_GUARDED_BY(rpc_lock()); // Successful calls to EncodeToPayloadBuffer MUST send the returned buffer, // without releasing the RPC lock. diff --git a/pw_rpc/py/pw_rpc/callback_client/call.py b/pw_rpc/py/pw_rpc/callback_client/call.py index 0378aeb30..6cb09aaef 100644 --- a/pw_rpc/py/pw_rpc/callback_client/call.py +++ b/pw_rpc/py/pw_rpc/callback_client/call.py @@ -46,17 +46,17 @@ class UseDefault(enum.Enum): VALUE = 0 -CallType = TypeVar( - 'CallType', +CallTypeT = TypeVar( + 'CallTypeT', 'UnaryCall', 'ServerStreamingCall', 'ClientStreamingCall', 'BidirectionalStreamingCall', ) -OnNextCallback = Callable[[CallType, Any], Any] -OnCompletedCallback = Callable[[CallType, Any], Any] -OnErrorCallback = Callable[[CallType, Any], Any] +OnNextCallback = Callable[[CallTypeT, Any], Any] +OnCompletedCallback = Callable[[CallTypeT, Any], Any] +OnErrorCallback = Callable[[CallTypeT, Any], Any] OptionalTimeout = Union[UseDefault, float, None] diff --git a/pw_rpc/py/pw_rpc/callback_client/impl.py b/pw_rpc/py/pw_rpc/callback_client/impl.py index e74e3043a..474757373 100644 --- a/pw_rpc/py/pw_rpc/callback_client/impl.py +++ b/pw_rpc/py/pw_rpc/callback_client/impl.py @@ -28,7 +28,7 @@ from pw_rpc.descriptors import Channel, Method, Service from pw_rpc.callback_client.call import ( UseDefault, OptionalTimeout, - CallType, + CallTypeT, UnaryResponse, StreamResponse, Call, @@ -106,14 +106,14 @@ class _MethodClient: def _start_call( self, - call_type: Type[CallType], + call_type: Type[CallTypeT], request: Optional[Message], timeout_s: OptionalTimeout, on_next: Optional[OnNextCallback], on_completed: Optional[OnCompletedCallback], on_error: Optional[OnErrorCallback], ignore_errors: bool = False, - ) -> CallType: + ) -> CallTypeT: """Creates the Call object and invokes the RPC using it.""" if timeout_s is UseDefault.VALUE: timeout_s = self.default_timeout_s @@ -125,8 +125,8 @@ class _MethodClient: return call def _client_streaming_call_type( - self, base: Type[CallType] - ) -> Type[CallType]: + self, base: Type[CallTypeT] + ) -> Type[CallTypeT]: """Creates a client or bidirectional stream call type. Applies the signature from the request protobuf to the send method. diff --git a/pw_rpc/py/pw_rpc/client.py b/pw_rpc/py/pw_rpc/client.py index 45dd98ecb..d8c3390d7 100644 --- a/pw_rpc/py/pw_rpc/client.py +++ b/pw_rpc/py/pw_rpc/client.py @@ -151,11 +151,11 @@ class PendingRpcs: packets.encode_client_stream_end(rpc) ) - def cancel(self, rpc: PendingRpc) -> Optional[bytes]: - """Cancels the RPC. Returns the CANCEL packet to send. + def cancel(self, rpc: PendingRpc) -> bytes: + """Cancels the RPC. Returns: - True if the RPC was cancelled; False if it was not pending + The CLIENT_ERROR packet to send. Raises: KeyError if the RPC is not pending @@ -163,9 +163,6 @@ class PendingRpcs: _LOG.debug('Cancelling %s', rpc) del self._pending[rpc] - if rpc.method.type is Method.Type.UNARY: - return None - return packets.encode_cancel(rpc) def send_cancel(self, rpc: PendingRpc) -> bool: @@ -400,12 +397,6 @@ def _update_for_backwards_compatibility( if rpc.method.type is not Method.Type.SERVER_STREAMING: return - # SERVER_STREAM_END packets are deprecated. They are equivalent to a - # RESPONSE packet. - if packet.type == PacketType.DEPRECATED_SERVER_STREAM_END: - packet.type = PacketType.RESPONSE - return - # Prior to the introduction of SERVER_STREAM packets, RESPONSE packets with # a payload were used instead. If a non-zero payload is present, assume this # RESPONSE is equivalent to a SERVER_STREAM packet. diff --git a/pw_rpc/py/pw_rpc/codegen_raw.py b/pw_rpc/py/pw_rpc/codegen_raw.py index 4e91dff75..01e03466a 100644 --- a/pw_rpc/py/pw_rpc/codegen_raw.py +++ b/pw_rpc/py/pw_rpc/codegen_raw.py @@ -185,7 +185,6 @@ class StubGenerator(codegen.StubGenerator): def server_streaming_signature( self, method: ProtoServiceMethod, prefix: str ) -> str: - return ( f'void {prefix}{method.name()}(' 'pw::ConstByteSpan request, RawServerWriter& writer)' diff --git a/pw_rpc/py/pw_rpc/lossy_channel.py b/pw_rpc/py/pw_rpc/lossy_channel.py index feb46c1d4..db8908a07 100644 --- a/pw_rpc/py/pw_rpc/lossy_channel.py +++ b/pw_rpc/py/pw_rpc/lossy_channel.py @@ -112,27 +112,22 @@ class ManualPacketFilter(LossController): def next_packet_duplicated(self) -> bool: return False - @staticmethod - def next_packet_out_of_order() -> bool: + def next_packet_out_of_order(self) -> bool: return False - @staticmethod - def next_packet_delayed() -> bool: + def next_packet_delayed(self) -> bool: return False def next_packet_dropped(self) -> bool: return not self.keep_packet() - @staticmethod - def next_packet_delay() -> int: + def next_packet_delay(self) -> int: return 0 - @staticmethod - def next_num_dupes() -> int: + def next_num_dupes(self) -> int: return 0 - @staticmethod - def choose_out_of_order_packet(max_idx) -> int: + def choose_out_of_order_packet(self, max_idx) -> int: return 0 diff --git a/pw_rpc/py/pw_rpc/testing.py b/pw_rpc/py/pw_rpc/testing.py index 0f1475899..de8ab6223 100644 --- a/pw_rpc/py/pw_rpc/testing.py +++ b/pw_rpc/py/pw_rpc/testing.py @@ -14,6 +14,7 @@ """Utilities for testing pw_rpc.""" import argparse +import shlex import subprocess import sys import tempfile @@ -68,8 +69,22 @@ def _parse_subprocess_integration_test_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description='Executes a test between two subprocesses' ) - parser.add_argument('--client', required=True, help='Client binary to run') - parser.add_argument('--server', required=True, help='Server binary to run') + parser.add_argument( + '--client', + required=True, + help=( + 'Client command to run. ' + 'Use quotes and whitespace to pass client-specifc arguments.' + ), + ) + parser.add_argument( + '--server', + required=True, + help=( + 'Server command to run. ' + 'Use quotes and whitespace to pass client-specifc arguments.' + ), + ) parser.add_argument( 'common_args', metavar='-- ...', @@ -96,6 +111,7 @@ def execute_integration_test( common_args: Sequence[str], setup_time_s: float = 0.2, ) -> int: + """Runs an RPC server and client as part of an integration test.""" temp_dir: Optional[tempfile.TemporaryDirectory] = None if TEMP_DIR_MARKER in common_args: @@ -105,11 +121,17 @@ def execute_integration_test( ] try: - server_process = subprocess.Popen([server, *common_args]) + server_cmdline = shlex.split(server) + client_cmdline = shlex.split(client) + if common_args: + server_cmdline += [*common_args] + client_cmdline += [*common_args] + + server_process = subprocess.Popen(server_cmdline) # TODO(b/234879791): Replace this delay with some sort of IPC. time.sleep(setup_time_s) - result = subprocess.run([client, *common_args]).returncode + result = subprocess.run(client_cmdline).returncode server_process.terminate() server_process.communicate() diff --git a/pw_rpc/py/tests/callback_client_test.py b/pw_rpc/py/tests/callback_client_test.py index 0f749613f..406182983 100755 --- a/pw_rpc/py/tests/callback_client_test.py +++ b/pw_rpc/py/tests/callback_client_test.py @@ -421,8 +421,10 @@ class UnaryTest(_CallbackClientImplTestBase): self.assertTrue(call.cancel()) self.assertFalse(call.cancel()) # Already cancelled, returns False - # Unary RPCs do not send a cancel request to the server. - self.assertFalse(self.requests) + self.assertEqual( + self.last_request().type, packet_pb2.PacketType.CLIENT_ERROR + ) + self.assertEqual(self.last_request().status, Status.CANCELLED.value) callback.assert_not_called() @@ -501,38 +503,6 @@ class ServerStreamingTest(_CallbackClientImplTestBase): 4, self._sent_payload(self.method.request_type).magic_number ) - def test_deprecated_packet_format(self) -> None: - rep1 = self.method.response_type(payload='!!!') - rep2 = self.method.response_type(payload='?') - - for _ in range(3): - # The original packet format used RESPONSE packets for the server - # stream and a SERVER_STREAM_END packet as the last packet. These - # are converted to SERVER_STREAM packets followed by a RESPONSE. - self._enqueue_response(1, self.method, payload=rep1) - self._enqueue_response(1, self.method, payload=rep2) - - self._next_packets.append( - ( - packet_pb2.RpcPacket( - type=packet_pb2.PacketType.DEPRECATED_SERVER_STREAM_END, - channel_id=1, - service_id=self.method.service.id, - method_id=self.method.id, - status=Status.INVALID_ARGUMENT.value, - ).SerializeToString(), - Status.OK, - ) - ) - - status, replies = self._service.SomeServerStreaming(magic_number=4) - self.assertEqual([rep1, rep2], replies) - self.assertIs(status, Status.INVALID_ARGUMENT) - - self.assertEqual( - 4, self._sent_payload(self.method.request_type).magic_number - ) - def test_nonblocking_call(self) -> None: rep1 = self.method.response_type(payload='!!!') rep2 = self.method.response_type(payload='?') diff --git a/pw_rpc/server.cc b/pw_rpc/server.cc index b953424d0..4af9ff4e6 100644 --- a/pw_rpc/server.cc +++ b/pw_rpc/server.cc @@ -86,7 +86,6 @@ Status Server::ProcessPacket(ConstByteSpan packet_data) { HandleClientStreamPacket(packet, *channel, call); break; case PacketType::CLIENT_ERROR: - case PacketType::DEPRECATED_CANCEL: if (call != calls_end()) { call->HandleError(packet.status()); } else { @@ -98,7 +97,6 @@ Status Server::ProcessPacket(ConstByteSpan packet_data) { break; case PacketType::REQUEST: // Handled above case PacketType::RESPONSE: - case PacketType::DEPRECATED_SERVER_STREAM_END: case PacketType::SERVER_ERROR: case PacketType::SERVER_STREAM: default: diff --git a/pw_rpc/ts/client_test.ts b/pw_rpc/ts/client_test.ts index dba716c2b..0dfed21b1 100644 --- a/pw_rpc/ts/client_test.ts +++ b/pw_rpc/ts/client_test.ts @@ -361,6 +361,9 @@ describe('RPC', () => { requests = []; expect(call.cancel()).toBe(true); + expect(lastRequest().getType()).toEqual(PacketType.CLIENT_ERROR); + expect(lastRequest().getStatus()).toEqual(Status.CANCELLED); + expect(call.cancel()).toBe(false); expect(onNext).not.toHaveBeenCalled(); } diff --git a/pw_rpc/ts/rpc_classes.ts b/pw_rpc/ts/rpc_classes.ts index 2cd62ce87..703612e19 100644 --- a/pw_rpc/ts/rpc_classes.ts +++ b/pw_rpc/ts/rpc_classes.ts @@ -112,13 +112,10 @@ export class PendingCalls { rpc.channel.send(packets.encodeClientStreamEnd(rpc.idSet)); } - /** Cancels the RPC. Returns the CANCEL packet to send. */ - cancel(rpc: Rpc): Uint8Array | undefined { + /** Cancels the RPC. Returns the CLIENT_ERROR packet to send. */ + cancel(rpc: Rpc): Uint8Array { console.debug(`Cancelling ${rpc}`); this.pending.delete(rpc.idString); - if (rpc.method.clientStreaming && rpc.method.serverStreaming) { - return undefined; - } return packets.encodeCancel(rpc.idSet); } diff --git a/pw_rust/examples/host_executable/BUILD.gn b/pw_rust/examples/host_executable/BUILD.gn index 99013eccd..fca1f74e1 100644 --- a/pw_rust/examples/host_executable/BUILD.gn +++ b/pw_rust/examples/host_executable/BUILD.gn @@ -16,6 +16,33 @@ import("//build_overrides/pigweed.gni") import("$dir_pw_build/target_types.gni") -pw_executable("hello") { - sources = [ "main.rs" ] +pw_rust_executable("hello") { + sources = [ + "main.rs", + "other.rs", + ] + + deps = [ + ":a", + ":c", + ] +} + +# The dep chain hello->a->b will exercise the functionality of both direct and +# transitive deps for A +pw_rust_library("a") { + crate_root = "a/lib.rs" + sources = [ "a/lib.rs" ] + deps = [ ":b" ] +} + +pw_rust_library("b") { + crate_root = "b/lib.rs" + sources = [ "b/lib.rs" ] + deps = [ ":c" ] +} + +pw_rust_library("c") { + crate_root = "c/lib.rs" + sources = [ "c/lib.rs" ] } diff --git a/pw_rust/examples/host_executable/a/lib.rs b/pw_rust/examples/host_executable/a/lib.rs new file mode 100644 index 000000000..e49ddb40a --- /dev/null +++ b/pw_rust/examples/host_executable/a/lib.rs @@ -0,0 +1,22 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#![warn(clippy::all)] + +use b::RequiredB; + +#[derive(Copy, Clone, Default)] +pub struct RequiredA { + pub required_b: RequiredB, +} diff --git a/pw_rust/examples/host_executable/b/lib.rs b/pw_rust/examples/host_executable/b/lib.rs new file mode 100644 index 000000000..6e32a1425 --- /dev/null +++ b/pw_rust/examples/host_executable/b/lib.rs @@ -0,0 +1,20 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#![warn(clippy::all)] + +#[derive(Copy, Clone, Debug, Default)] +pub struct RequiredB { + pub value: i32, +} diff --git a/pw_rust/examples/host_executable/c/lib.rs b/pw_rust/examples/host_executable/c/lib.rs new file mode 100644 index 000000000..3c2cbff78 --- /dev/null +++ b/pw_rust/examples/host_executable/c/lib.rs @@ -0,0 +1,19 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#![warn(clippy::all)] + +pub fn value() -> i32 { + 1 +} diff --git a/pw_rust/examples/host_executable/main.rs b/pw_rust/examples/host_executable/main.rs index b78a75216..79d72df70 100644 --- a/pw_rust/examples/host_executable/main.rs +++ b/pw_rust/examples/host_executable/main.rs @@ -1,4 +1,4 @@ -// Copyright 2022 The Pigweed Authors +// Copyright 2023 The Pigweed Authors // // Licensed under the Apache License, Version 2.0 (the "License"); you may not // use this file except in compliance with the License. You may obtain a copy of @@ -12,6 +12,31 @@ // License for the specific language governing permissions and limitations under // the License. +#![warn(clippy::all)] + +mod other; + fn main() { println!("Hello, Pigweed!"); + + // ensure we can run code from other modules in the main crate + println!("{}", other::foo()); + + // ensure we can run code from dependent libraries + println!("{}", a::RequiredA::default().required_b.value); + println!("{}", c::value()); +} + +#[cfg(test)] +mod tests { + #[test] + fn test_simple() { + let x = 3.14; + assert!(x > 0.0); + } + + #[test] + fn test_other_module() { + assert!(a::RequiredA::default().required_b.value == 0); + } } diff --git a/pw_rust/examples/host_executable/other.rs b/pw_rust/examples/host_executable/other.rs new file mode 100644 index 000000000..fd571096f --- /dev/null +++ b/pw_rust/examples/host_executable/other.rs @@ -0,0 +1,18 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#[allow(unused)] +pub fn foo() -> i32 { + return 42; +} diff --git a/pw_software_update/py/pw_software_update/cli.py b/pw_software_update/py/pw_software_update/cli.py index d49993904..a099beff5 100644 --- a/pw_software_update/py/pw_software_update/cli.py +++ b/pw_software_update/py/pw_software_update/cli.py @@ -198,7 +198,7 @@ def add_file_to_bundle_handler(arg) -> None: arg.bundle.write_bytes(updated_bundle.SerializeToString()) - except (IOError) as error: + except IOError as error: print(error) diff --git a/pw_software_update/py/pw_software_update/generate_test_bundle.py b/pw_software_update/py/pw_software_update/generate_test_bundle.py index 0c52439df..14409ae7d 100644 --- a/pw_software_update/py/pw_software_update/generate_test_bundle.py +++ b/pw_software_update/py/pw_software_update/generate_test_bundle.py @@ -280,11 +280,11 @@ def parse_args(): return parser.parse_args() +# TODO(b/237580538): Refactor the code so that each test bundle generation +# is done in a separate function or script. +# pylint: disable=too-many-locals def main() -> int: """Main""" - # TODO(b/237580538): Refactor the code so that each test bundle generation - # is done in a separate function or script. - # pylint: disable=too-many-locals args = parse_args() test_bundle = Bundle() @@ -515,11 +515,13 @@ def main() -> int: ], check=True, ) - # TODO(b/237580538): Refactor the code so that each test bundle generation - # is done in a separate function or script. - # pylint: enable=too-many-locals return 0 +# TODO(b/237580538): Refactor the code so that each test bundle generation +# is done in a separate function or script. +# pylint: enable=too-many-locals + + if __name__ == "__main__": sys.exit(main()) diff --git a/pw_software_update/py/pw_software_update/metadata.py b/pw_software_update/py/pw_software_update/metadata.py index 86def4995..0816d7cc8 100644 --- a/pw_software_update/py/pw_software_update/metadata.py +++ b/pw_software_update/py/pw_software_update/metadata.py @@ -54,7 +54,6 @@ def gen_common_metadata( def gen_target_file( file_name: str, file_contents: bytes, hash_funcs=DEFAULT_HASHES ) -> TargetFile: - return TargetFile( file_name=file_name, length=len(file_contents), diff --git a/pw_software_update/update_bundle_test.cc b/pw_software_update/update_bundle_test.cc index c2bbf660e..e35987a38 100644 --- a/pw_software_update/update_bundle_test.cc +++ b/pw_software_update/update_bundle_test.cc @@ -69,7 +69,7 @@ class TestBundledUpdateBackend final : public BundledUpdateBackend { void SetManifestWriter(stream::Writer* writer) { manifest_writer_ = writer; } - virtual Result<stream::SeekableReader*> GetRootMetadataReader() override { + Result<stream::SeekableReader*> GetRootMetadataReader() override { return &trusted_root_reader_; } @@ -105,7 +105,7 @@ class TestBundledUpdateBackend final : public BundledUpdateBackend { return manifest_writer_; } - virtual Status SafelyPersistRootMetadata( + Status SafelyPersistRootMetadata( [[maybe_unused]] stream::IntervalReader root_metadata) override { new_root_persisted_ = true; trusted_root_reader_ = root_metadata; @@ -273,7 +273,7 @@ TEST_F(UpdateBundleTest, PersistManifestFailIfNotVerified) { TEST_F(UpdateBundleTest, SelfVerificationWithIncomingRoot) { StageTestBundle(kTestDevBundleWithRoot); UpdateBundleAccessor update_bundle( - blob_reader(), backend(), /* disable_verification = */ true); + blob_reader(), backend(), /* self_verification = */ true); ASSERT_OK(update_bundle.OpenAndVerify()); // Self verification must not persist anything. @@ -293,7 +293,7 @@ TEST_F(UpdateBundleTest, SelfVerificationWithIncomingRoot) { TEST_F(UpdateBundleTest, SelfVerificationWithoutIncomingRoot) { StageTestBundle(kTestDevBundle); UpdateBundleAccessor update_bundle( - blob_reader(), backend(), /* disable_verification = */ true); + blob_reader(), backend(), /* self_verification = */ true); ASSERT_OK(update_bundle.OpenAndVerify()); } @@ -301,7 +301,7 @@ TEST_F(UpdateBundleTest, SelfVerificationWithoutIncomingRoot) { TEST_F(UpdateBundleTest, SelfVerificationWithMessedUpRoot) { StageTestBundle(kTestDevBundleWithProdRoot); UpdateBundleAccessor update_bundle( - blob_reader(), backend(), /* disable_verification = */ true); + blob_reader(), backend(), /* self_verification = */ true); ASSERT_FAIL(update_bundle.OpenAndVerify()); } @@ -309,7 +309,7 @@ TEST_F(UpdateBundleTest, SelfVerificationWithMessedUpRoot) { TEST_F(UpdateBundleTest, SelfVerificationChecksMissingHashes) { StageTestBundle(kTestBundleMissingTargetHashFile0); UpdateBundleAccessor update_bundle( - blob_reader(), backend(), /* disable_verification = */ true); + blob_reader(), backend(), /* self_verification = */ true); ASSERT_FAIL(update_bundle.OpenAndVerify()); } @@ -317,7 +317,7 @@ TEST_F(UpdateBundleTest, SelfVerificationChecksMissingHashes) { TEST_F(UpdateBundleTest, SelfVerificationChecksBadHashes) { StageTestBundle(kTestBundleMismatchedTargetHashFile0); UpdateBundleAccessor update_bundle( - blob_reader(), backend(), /* disable_verification = */ true); + blob_reader(), backend(), /* self_verification = */ true); ASSERT_FAIL(update_bundle.OpenAndVerify()); } @@ -325,7 +325,7 @@ TEST_F(UpdateBundleTest, SelfVerificationChecksBadHashes) { TEST_F(UpdateBundleTest, SelfVerificationIgnoresUnsignedBundle) { StageTestBundle(kTestUnsignedBundleWithRoot); UpdateBundleAccessor update_bundle( - blob_reader(), backend(), /* disable_verification = */ true); + blob_reader(), backend(), /* self_verification = */ true); ASSERT_OK(update_bundle.OpenAndVerify()); } diff --git a/pw_stream/BUILD.bazel b/pw_stream/BUILD.bazel index c4d910ade..524e2082a 100644 --- a/pw_stream/BUILD.bazel +++ b/pw_stream/BUILD.bazel @@ -51,6 +51,7 @@ pw_cc_library( deps = [ ":pw_stream", "//pw_log", + "//pw_string", "//pw_sys_io", ], ) @@ -143,3 +144,12 @@ pw_cc_test( "//pw_unit_test", ], ) + +pw_cc_test( + name = "socket_stream_test", + srcs = ["socket_stream_test.cc"], + deps = [ + ":socket_stream", + "//pw_unit_test", + ], +) diff --git a/pw_stream/BUILD.gn b/pw_stream/BUILD.gn index 1ac95a348..9f4eb2f8f 100644 --- a/pw_stream/BUILD.gn +++ b/pw_stream/BUILD.gn @@ -46,7 +46,11 @@ pw_source_set("pw_stream") { pw_source_set("socket_stream") { public_configs = [ ":public_include_path" ] public_deps = [ ":pw_stream" ] - deps = [ dir_pw_log ] + deps = [ + dir_pw_assert, + dir_pw_log, + dir_pw_string, + ] sources = [ "socket_stream.cc" ] public = [ "public/pw_stream/socket_stream.h" ] } @@ -95,6 +99,11 @@ pw_test_group("tests") { if (defined(pw_toolchain_SCOPE.is_host_toolchain) && pw_toolchain_SCOPE.is_host_toolchain) { tests += [ ":std_file_stream_test" ] + + # socket_stream_test doesn't compile on Windows. + if (host_os != "win") { + tests += [ ":socket_stream_test" ] + } } } @@ -136,3 +145,8 @@ pw_test("interval_reader_test") { sources = [ "interval_reader_test.cc" ] deps = [ ":interval_reader" ] } + +pw_test("socket_stream_test") { + sources = [ "socket_stream_test.cc" ] + deps = [ ":socket_stream" ] +} diff --git a/pw_stream/CMakeLists.txt b/pw_stream/CMakeLists.txt index 5f71629e0..dfa45823f 100644 --- a/pw_stream/CMakeLists.txt +++ b/pw_stream/CMakeLists.txt @@ -47,6 +47,7 @@ pw_add_library(pw_stream.socket_stream STATIC socket_stream.cc PRIVATE_DEPS pw_log + pw_string ) pw_add_library(pw_stream.sys_io_stream INTERFACE diff --git a/pw_stream/docs.rst b/pw_stream/docs.rst index dd4fef167..a4aaeb913 100644 --- a/pw_stream/docs.rst +++ b/pw_stream/docs.rst @@ -209,7 +209,7 @@ Reader interfaces ----------------- .. cpp:class:: Reader : public Stream - A Stream that supports writing but not reading. The Write() method is hidden. + A Stream that supports reading but not writing. The Write() method is hidden. Use in APIs when: * Must read from, but not write to, a stream. @@ -426,8 +426,14 @@ Implementations .. cpp:class:: SocketStream : public NonSeekableReaderWriter - ``SocketStream`` wraps posix-style sockets with the :cpp:class:`Reader` and - :cpp:class:`Writer` interfaces. + ``SocketStream`` wraps posix-style TCP sockets with the :cpp:class:`Reader` + and :cpp:class:`Writer` interfaces. It can be used to connect to a TCP server, + or to communicate with a client via the ``ServerSocket`` class. + +.. cpp:class:: ServerSocket + + ``ServerSocket`` wraps a posix server socket, and produces a + :cpp:class:`SocketStream` for each accepted client connection. ------------------ Why use pw_stream? diff --git a/pw_stream/public/pw_stream/socket_stream.h b/pw_stream/public/pw_stream/socket_stream.h index d7578221c..0092fc808 100644 --- a/pw_stream/public/pw_stream/socket_stream.h +++ b/pw_stream/public/pw_stream/socket_stream.h @@ -17,6 +17,7 @@ #include <cstdint> +#include "pw_result/result.h" #include "pw_span/span.h" #include "pw_stream/stream.h" @@ -26,13 +27,38 @@ class SocketStream : public NonSeekableReaderWriter { public: constexpr SocketStream() = default; + // SocketStream objects are moveable but not copyable. + SocketStream& operator=(SocketStream&& other) { + listen_port_ = other.listen_port_; + socket_fd_ = other.socket_fd_; + other.socket_fd_ = kInvalidFd; + connection_fd_ = other.connection_fd_; + other.connection_fd_ = kInvalidFd; + sockaddr_client_ = other.sockaddr_client_; + return *this; + } + SocketStream(SocketStream&& other) noexcept + : listen_port_(other.listen_port_), + socket_fd_(other.socket_fd_), + connection_fd_(other.connection_fd_), + sockaddr_client_(other.sockaddr_client_) { + other.socket_fd_ = kInvalidFd; + other.connection_fd_ = kInvalidFd; + } + SocketStream(const SocketStream&) = delete; + SocketStream& operator=(const SocketStream&) = delete; + ~SocketStream() override { Close(); } // Listen to the port and return after a client is connected + // + // DEPRECATED: Use the ServerSocket class instead. + // TODO(b/271323032): Remove when this method is no longer used. Status Serve(uint16_t port); - // Connect to a local or remote endpoint. Host must be an IPv4 address. If - // host is nullptr then the locahost address is used instead. + // Connect to a local or remote endpoint. Host may be either an IPv4 or IPv6 + // address. If host is nullptr then the IPv4 localhost address is used + // instead. Status Connect(const char* host, uint16_t port); // Close the socket stream and release all resources @@ -46,6 +72,8 @@ class SocketStream : public NonSeekableReaderWriter { int connection_fd() { return connection_fd_; } private: + friend class ServerSocket; + static constexpr int kInvalidFd = -1; Status DoWrite(span<const std::byte> data) override; @@ -58,4 +86,39 @@ class SocketStream : public NonSeekableReaderWriter { struct sockaddr_in sockaddr_client_ = {}; }; +/// `ServerSocket` wraps a POSIX-style server socket, producing a `SocketStream` +/// for each accepted client connection. +/// +/// Call `Listen` to create the socket and start listening for connections. +/// Then call `Accept` any number of times to accept client connections. +class ServerSocket { + public: + ServerSocket() = default; + ~ServerSocket() { Close(); } + + ServerSocket(const ServerSocket& other) = delete; + ServerSocket& operator=(const ServerSocket& other) = delete; + + // Listen for connections on the given port. + // If port is 0, a random unused port is chosen and can be retrieved with + // port(). + Status Listen(uint16_t port = 0); + + // Accept a connection. Blocks until after a client is connected. + // On success, returns a SocketStream connected to the new client. + Result<SocketStream> Accept(); + + // Close the server socket, preventing further connections. + void Close(); + + // Returns the port this socket is listening on. + uint16_t port() const { return port_; } + + private: + static constexpr int kInvalidFd = -1; + + uint16_t port_ = -1; + int socket_fd_ = kInvalidFd; +}; + } // namespace pw::stream diff --git a/pw_stream/socket_stream.cc b/pw_stream/socket_stream.cc index 564857b81..3978e49ab 100644 --- a/pw_stream/socket_stream.cc +++ b/pw_stream/socket_stream.cc @@ -15,18 +15,37 @@ #include "pw_stream/socket_stream.h" #include <arpa/inet.h> +#include <netdb.h> +#include <netinet/in.h> +#include <sys/socket.h> +#include <sys/types.h> #include <unistd.h> +#include <cerrno> #include <cstring> +#include "pw_assert/check.h" #include "pw_log/log.h" +#include "pw_string/to_string.h" namespace pw::stream { namespace { -constexpr uint32_t kMaxConcurrentUser = 1; +constexpr uint32_t kServerBacklogLength = 1; constexpr const char* kLocalhostAddress = "127.0.0.1"; +// Set necessary options on a socket file descriptor. +void ConfigureSocket([[maybe_unused]] int socket) { +#if defined(__APPLE__) + // Use SO_NOSIGPIPE to avoid getting a SIGPIPE signal when the remote peer + // drops the connection. This is supported on macOS only. + constexpr int value = 1; + if (setsockopt(socket, SOL_SOCKET, SO_NOSIGPIPE, &value, sizeof(int)) < 0) { + PW_LOG_WARN("Failed to set SO_NOSIGPIPE: %s", std::strerror(errno)); + } +#endif // defined(__APPLE__) +} + } // namespace // TODO(b/240982565): Implement SocketStream for Windows. @@ -65,7 +84,7 @@ Status SocketStream::Serve(uint16_t port) { return Status::Unknown(); } - if (listen(socket_fd_, kMaxConcurrentUser) < 0) { + if (listen(socket_fd_, kServerBacklogLength) < 0) { PW_LOG_ERROR("Failed to listen to socket: %s", std::strerror(errno)); return Status::Unknown(); } @@ -77,28 +96,36 @@ Status SocketStream::Serve(uint16_t port) { if (connection_fd_ < 0) { return Status::Unknown(); } + ConfigureSocket(connection_fd_); return OkStatus(); } Status SocketStream::SocketStream::Connect(const char* host, uint16_t port) { - connection_fd_ = socket(AF_INET, SOCK_STREAM, 0); - - sockaddr_in addr; - addr.sin_family = AF_INET; - addr.sin_port = htons(port); - if (host == nullptr || std::strcmp(host, "localhost") == 0) { host = kLocalhostAddress; } - if (inet_pton(AF_INET, host, &addr.sin_addr) <= 0) { + struct addrinfo hints = {}; + struct addrinfo* res; + char port_buffer[6]; + PW_CHECK(ToString(port, port_buffer).ok()); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_NUMERICHOST | AI_NUMERICSERV | AI_PASSIVE; + if (getaddrinfo(host, port_buffer, &hints, &res) != 0) { PW_LOG_ERROR("Failed to configure connection address for socket"); return Status::InvalidArgument(); } - if (connect(connection_fd_, - reinterpret_cast<sockaddr*>(&addr), - sizeof(addr)) < 0) { + connection_fd_ = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + ConfigureSocket(connection_fd_); + if (connect(connection_fd_, res->ai_addr, res->ai_addrlen) < 0) { + close(connection_fd_); + connection_fd_ = kInvalidFd; + } + freeaddrinfo(res); + + if (connection_fd_ == kInvalidFd) { PW_LOG_ERROR( "Failed to connect to %s:%d: %s", host, port, std::strerror(errno)); return Status::Unknown(); @@ -120,10 +147,15 @@ void SocketStream::Close() { } Status SocketStream::DoWrite(span<const std::byte> data) { + int send_flags = 0; +#if defined(__linux__) // Use MSG_NOSIGNAL to avoid getting a SIGPIPE signal when the remote - // peer drops the connection. + // peer drops the connection. This is supported on Linux only. + send_flags |= MSG_NOSIGNAL; +#endif // defined(__linux__) + ssize_t bytes_sent = - send(connection_fd_, data.data(), data.size_bytes(), MSG_NOSIGNAL); + send(connection_fd_, data.data(), data.size_bytes(), send_flags); if (bytes_sent < 0 || static_cast<size_t>(bytes_sent) != data.size()) { if (errno == EPIPE) { @@ -156,4 +188,73 @@ StatusWithSize SocketStream::DoRead(ByteSpan dest) { return StatusWithSize(bytes_rcvd); } +// Listen for connections on the given port. +// If port is 0, a random unused port is chosen and can be retrieved with +// port(). +Status ServerSocket::Listen(uint16_t port) { + socket_fd_ = socket(AF_INET6, SOCK_STREAM, 0); + if (socket_fd_ == kInvalidFd) { + return Status::Unknown(); + } + + // Allow binding to an address that may still be in use by a closed socket. + constexpr int value = 1; + setsockopt(socket_fd_, SOL_SOCKET, SO_REUSEADDR, &value, sizeof(int)); + + if (port != 0) { + struct sockaddr_in6 addr = {}; + socklen_t addr_len = sizeof(addr); + addr.sin6_family = AF_INET6; + addr.sin6_port = htons(port); + addr.sin6_addr = in6addr_any; + if (bind(socket_fd_, reinterpret_cast<sockaddr*>(&addr), addr_len) < 0) { + return Status::Unknown(); + } + } + + if (listen(socket_fd_, kServerBacklogLength) < 0) { + return Status::Unknown(); + } + + // Find out which port the socket is listening on, and fill in port_. + struct sockaddr_in6 addr = {}; + socklen_t addr_len = sizeof(addr); + if (getsockname(socket_fd_, reinterpret_cast<sockaddr*>(&addr), &addr_len) < + 0 || + addr_len > sizeof(addr)) { + close(socket_fd_); + return Status::Unknown(); + } + + port_ = ntohs(addr.sin6_port); + + return OkStatus(); +} + +// Accept a connection. Blocks until after a client is connected. +// On success, returns a SocketStream connected to the new client. +Result<SocketStream> ServerSocket::Accept() { + struct sockaddr_in6 sockaddr_client_ = {}; + socklen_t len = sizeof(sockaddr_client_); + + int connection_fd = + accept(socket_fd_, reinterpret_cast<sockaddr*>(&sockaddr_client_), &len); + if (connection_fd == kInvalidFd) { + return Status::Unknown(); + } + ConfigureSocket(connection_fd); + + SocketStream client_stream; + client_stream.connection_fd_ = connection_fd; + return client_stream; +} + +// Close the server socket, preventing further connections. +void ServerSocket::Close() { + if (socket_fd_ != kInvalidFd) { + close(socket_fd_); + socket_fd_ = kInvalidFd; + } +} + } // namespace pw::stream diff --git a/pw_stream/socket_stream_test.cc b/pw_stream/socket_stream_test.cc new file mode 100644 index 000000000..bd8e58eae --- /dev/null +++ b/pw_stream/socket_stream_test.cc @@ -0,0 +1,194 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#include "pw_stream/socket_stream.h" + +#include <thread> + +#include "gtest/gtest.h" +#include "pw_result/result.h" +#include "pw_status/status.h" + +namespace pw::stream { +namespace { + +// Helper function to create a ServerSocket and connect to it via loopback. +void RunConnectTest(const char* hostname) { + ServerSocket server; + EXPECT_EQ(server.Listen(), OkStatus()); + + Result<SocketStream> server_stream = Status::Unavailable(); + auto accept_thread = std::thread{[&]() { server_stream = server.Accept(); }}; + + SocketStream client; + EXPECT_EQ(client.Connect(hostname, server.port()), OkStatus()); + + accept_thread.join(); + EXPECT_EQ(server_stream.status(), OkStatus()); + + server_stream.value().Close(); + server.Close(); + client.Close(); +} + +TEST(SocketStreamTest, ConnectIpv4) { RunConnectTest("127.0.0.1"); } + +TEST(SocketStreamTest, ConnectIpv6) { RunConnectTest("::1"); } + +TEST(SocketStreamTest, ConnectSpecificPort) { + // We want to test the "listen on a specific port" functionality, + // but hard-coding a port number in a test is inherently problematic, as + // port numbers are a global resource. + // + // We use the automatic port assignment initially to get a port assignment, + // close that server, and then use that port explicitly in a new server. + // + // There's still the possibility that the port will get swiped, but it + // shouldn't happen by chance. + ServerSocket initial_server; + EXPECT_EQ(initial_server.Listen(), OkStatus()); + uint16_t port = initial_server.port(); + initial_server.Close(); + + ServerSocket server; + EXPECT_EQ(server.Listen(port), OkStatus()); + EXPECT_EQ(server.port(), port); + + Result<SocketStream> server_stream = Status::Unavailable(); + auto accept_thread = std::thread{[&]() { server_stream = server.Accept(); }}; + + SocketStream client; + EXPECT_EQ(client.Connect("localhost", server.port()), OkStatus()); + + accept_thread.join(); + EXPECT_EQ(server_stream.status(), OkStatus()); + + server_stream.value().Close(); + server.Close(); + client.Close(); +} + +// Helper function to test exchanging data on a pair of sockets. +void ExchangeData(SocketStream& stream1, SocketStream& stream2) { + auto kPayload1 = as_bytes(span("some data")); + auto kPayload2 = as_bytes(span("other bytes")); + std::array<char, 100> read_buffer{}; + + // Write data from stream1 and read it from stream2. + auto write_status = Status::Unavailable(); + auto write_thread = + std::thread{[&]() { write_status = stream1.Write(kPayload1); }}; + Result<ByteSpan> read_result = + stream2.Read(as_writable_bytes(span(read_buffer))); + EXPECT_EQ(read_result.status(), OkStatus()); + EXPECT_EQ(read_result.value().size(), kPayload1.size()); + EXPECT_TRUE( + std::equal(kPayload1.begin(), kPayload1.end(), read_result->begin())); + + write_thread.join(); + EXPECT_EQ(write_status, OkStatus()); + + // Read data in the client and write it from the server. + auto read_thread = std::thread{[&]() { + read_result = stream1.Read(as_writable_bytes(span(read_buffer))); + }}; + EXPECT_EQ(stream2.Write(kPayload2), OkStatus()); + + read_thread.join(); + EXPECT_EQ(read_result.status(), OkStatus()); + EXPECT_EQ(read_result.value().size(), kPayload2.size()); + EXPECT_TRUE( + std::equal(kPayload2.begin(), kPayload2.end(), read_result->begin())); + + // Close stream1 and attempt to read from stream2. + stream1.Close(); + read_result = stream2.Read(as_writable_bytes(span(read_buffer))); + EXPECT_EQ(read_result.status(), Status::OutOfRange()); + + stream2.Close(); +} + +TEST(SocketStreamTest, ReadWrite) { + ServerSocket server; + EXPECT_EQ(server.Listen(), OkStatus()); + + Result<SocketStream> server_stream = Status::Unavailable(); + auto accept_thread = std::thread{[&]() { server_stream = server.Accept(); }}; + + SocketStream client; + EXPECT_EQ(client.Connect("localhost", server.port()), OkStatus()); + + accept_thread.join(); + EXPECT_EQ(server_stream.status(), OkStatus()); + + ExchangeData(client, server_stream.value()); + server.Close(); +} + +TEST(SocketStreamTest, MultipleClients) { + ServerSocket server; + EXPECT_EQ(server.Listen(), OkStatus()); + + Result<SocketStream> server_stream1 = Status::Unavailable(); + Result<SocketStream> server_stream2 = Status::Unavailable(); + Result<SocketStream> server_stream3 = Status::Unavailable(); + auto accept_thread = std::thread{[&]() { + server_stream1 = server.Accept(); + server_stream2 = server.Accept(); + server_stream3 = server.Accept(); + }}; + + SocketStream client1; + SocketStream client2; + SocketStream client3; + EXPECT_EQ(client1.Connect("localhost", server.port()), OkStatus()); + EXPECT_EQ(client2.Connect("localhost", server.port()), OkStatus()); + EXPECT_EQ(client3.Connect("localhost", server.port()), OkStatus()); + + accept_thread.join(); + EXPECT_EQ(server_stream1.status(), OkStatus()); + EXPECT_EQ(server_stream2.status(), OkStatus()); + EXPECT_EQ(server_stream3.status(), OkStatus()); + + ExchangeData(client1, server_stream1.value()); + ExchangeData(client2, server_stream2.value()); + ExchangeData(client3, server_stream3.value()); + server.Close(); +} + +TEST(SocketStreamTest, ReuseAutomaticServerPort) { + uint16_t server_port = 0; + SocketStream client_stream; + ServerSocket server; + + EXPECT_EQ(server.Listen(0), OkStatus()); + server_port = server.port(); + EXPECT_NE(server_port, 0); + + Result<SocketStream> server_stream = Status::Unavailable(); + auto accept_thread = std::thread{[&]() { server_stream = server.Accept(); }}; + + EXPECT_EQ(client_stream.Connect(nullptr, server_port), OkStatus()); + accept_thread.join(); + ASSERT_EQ(server_stream.status(), OkStatus()); + + server_stream->Close(); + server.Close(); + + ServerSocket server2; + EXPECT_EQ(server2.Listen(server_port), OkStatus()); +} + +} // namespace +} // namespace pw::stream diff --git a/pw_string/api.rst b/pw_string/api.rst index 3dc701df4..a526023eb 100644 --- a/pw_string/api.rst +++ b/pw_string/api.rst @@ -1,94 +1,71 @@ .. _module-pw_string-api: -======================= -pw_string API reference -======================= +============= +API Reference +============= -.. _pw_inlinebasicstring-api: +-------- +Overview +-------- +This module provides two types of strings, and utility functions for working +with strings. ---------------------- -pw::InlineBasicString ---------------------- -:cpp:class:`pw::InlineBasicString` and :cpp:type:`pw::InlineString` are -C++14-compatible, fixed-capacity, null-terminated string classes. They are -equivalent to ``std::basic_string<T>`` and ``std::string``, but store the string -contents inline and use no dynamic memory. +**pw::StringBuilder** -:cpp:type:`pw::InlineString` takes the fixed capacity as a template argument, -but may be used generically without specifying the capacity. The capacity value -is stored in a member variable, which the generic ``pw::InlineString<>`` / -``pw::InlineBasicString<T>`` specialization uses in place of the template -parameter. +.. doxygenfile:: pw_string/string_builder.h + :sections: briefdescription -:cpp:type:`pw::InlineString` is efficient and compact. The current size and -capacity are stored in a single word. Accessing the contents of a -:cpp:type:`pw::InlineString` is a simple array access within the object, with no -pointer indirection, even when working from a generic ``pw::InlineString<>`` -reference. +**pw::InlineString** -.. cpp:class:: template <typename T, unsigned short kCapacity> pw::InlineBasicString +.. doxygenfile:: pw_string/string.h + :sections: briefdescription - Represents a fixed-capacity string of a generic character type. Equivalent to - ``std::basic_string<T>``. Always null (``T()``) terminated. +**String utility functions** + +.. doxygenfile:: pw_string/util.h + :sections: briefdescription + +----------------- +pw::StringBuilder +----------------- +.. doxygenfile:: pw_string/string_builder.h + :sections: briefdescription +.. doxygenclass:: pw::StringBuilder + :members: ---------------- pw::InlineString ---------------- -See also :ref:`pw_inlinebasicstring-api`. - -.. cpp:type:: template <unsigned short kCapacity> pw::InlineString = pw::InlineBasicString<char, kCapacity> - - Represents a fixed-capacity string of ``char`` characters. Equivalent to - ``std::string``. Always null (``'\0'``) terminated. +.. doxygenfile:: pw_string/string.h + :sections: detaileddescription ----------- -pw::string ----------- - -pw::string::Assign ------------------- -.. cpp:function:: pw::Status pw::string::Assign(pw::InlineString<>& string, const std::string_view& view) - - Assigns ``view`` to ``string``. Truncates and returns ``RESOURCE_EXHAUSTED`` - if ``view`` is too large for ``string``. - -pw::string::Append ------------------- -.. cpp:function:: pw::Status pw::string::Append(pw::InlineString<>& string, const std::string_view& view) - - Appends ``view`` to ``string``. Truncates and returns ``RESOURCE_EXHAUSTED`` - if ``view`` does not fit within the remaining capacity of ``string``. - -pw::string::ClampedCString --------------------------- -.. cpp:function:: constexpr std::string_view pw::string::ClampedCString(span<const char> str) -.. cpp:function:: constexpr std::string_view pw::string::ClampedCString(const char* str, size_t max_len) +.. doxygenclass:: pw::InlineBasicString + :members: - Safe alternative to the string_view constructor to avoid the risk of an - unbounded implicit or explicit use of strlen. +.. doxygentypedef:: pw::InlineString - This is strongly recommended over using something like C11's strnlen_s as - a string_view does not require null-termination. +------------------------ +String utility functions +------------------------ -pw::string::Copy ----------------- -The ``pw::string::Copy`` functions provide a safer alternative to -``std::strncpy`` as it always null-terminates whenever the destination -buffer has a non-zero size. +pw::string::Assign() +-------------------- +.. doxygenfunction:: pw::string::Assign(InlineString<> &string, const std::string_view &view) -.. cpp:function:: StatusWithSize Copy(const std::string_view& source, span<char> dest) -.. cpp:function:: StatusWithSize Copy(const char* source, span<char> dest) -.. cpp:function:: StatusWithSize Copy(const char* source, char* dest, size_t num) -.. cpp:function:: StatusWithSize Copy(const pw::Vector<char>& source, span<char> dest) +pw::string::Append() +-------------------- +.. doxygenfunction:: pw::string::Append(InlineString<>& string, const std::string_view& view) - Copies the source string to the dest, truncating if the full string does not - fit. Always null terminates if dest.size() or num > 0. +pw::string::ClampedCString() +---------------------------- +.. doxygenfunction:: pw::string::ClampedCString(const char* str, size_t max_len) +.. doxygenfunction:: pw::string::ClampedCString(span<const char> str) - Returns the number of characters written, excluding the null terminator. If - the string is truncated, the status is ResourceExhausted. - - Precondition: The destination and source shall not overlap. - Precondition: The source shall be a valid pointer. +pw::string::Copy() +------------------ +.. doxygenfunction:: pw::string::Copy(const char* source, char* dest, size_t num) +.. doxygenfunction:: pw::string::Copy(const char* source, Span&& dest) +.. doxygenfunction:: pw::string::Copy(const std::string_view& source, Span&& dest) It also has variants that provide a destination of ``pw::Vector<char>`` (see :ref:`module-pw_containers` for details) that do not store the null @@ -97,43 +74,18 @@ terminator in the vector. .. cpp:function:: StatusWithSize Copy(const std::string_view& source, pw::Vector<char>& dest) .. cpp:function:: StatusWithSize Copy(const char* source, pw::Vector<char>& dest) -pw::string::NullTerminatedLength --------------------------------- -.. cpp:function:: constexpr pw::Result<size_t> pw::string::NullTerminatedLength(span<const char> str) -.. cpp:function:: pw::Result<size_t> pw::string::NullTerminatedLength(const char* str, size_t max_len) - - Safe alternative to strlen to calculate the null-terminated length of the - string within the specified span, excluding the null terminator. Like C11's - strnlen_s, the scan for the null-terminator is bounded. - - Returns: - null-terminated length of the string excluding the null terminator. - OutOfRange - if the string is not null-terminated. - - Precondition: The string shall be at a valid pointer. - -pw::string::PrintableCopy -------------------------- -The ``pw::string::PrintableCopy`` function provides a safe printable copy of a -string. It functions with the same safety of ``pw::string::Copy`` while also -converting any non-printable characters to a ``.`` char. - -.. cpp:function:: StatusWithSize PrintableCopy(const std::string_view& source, span<char> dest) - ------------------ -pw::StringBuilder ------------------ -.. cpp:namespace-push:: pw::StringBuilder - -:cpp:class:`StringBuilder` facilitates creating formatted strings in a -fixed-sized buffer or :cpp:type:`pw::InlineString`. It is designed to give the -flexibility of ``std::ostringstream``, but with a small footprint. - -:cpp:class:`StringBuilder` supports C++ ``<<``-style output, printf formatting, -and a few ``std::string`` functions (:cpp:func:`append()`, -:cpp:func:`push_back()`, :cpp:func:`pop_back`. - -.. cpp:namespace-pop:: - -.. doxygenclass:: pw::StringBuilder - :members: +pw::string::Format() +-------------------- +.. doxygenfile:: pw_string/format.h + :sections: briefdescription +.. doxygenfunction:: pw::string::Format +.. doxygenfunction:: pw::string::FormatVaList + +pw::string::NullTerminatedLength() +---------------------------------- +.. doxygenfunction:: pw::string::NullTerminatedLength(const char* str, size_t max_len) +.. doxygenfunction:: pw::string::NullTerminatedLength(span<const char> str) + +pw::string::PrintableCopy() +--------------------------- +.. doxygenfunction:: pw::string::PrintableCopy(const std::string_view& source, span<char> dest) diff --git a/pw_string/design.rst b/pw_string/design.rst index c84465cfb..91202e185 100644 --- a/pw_string/design.rst +++ b/pw_string/design.rst @@ -3,45 +3,65 @@ ================ pw_string design ================ -.. - This doc provides background on how a module works internally, the assumptions - inherent in its design, why this particular design was chosen over others, and - other topics of that nature. - -:cpp:type:`pw::InlineString` / :cpp:class:`pw::InlineBasicString` follows the -``std::string`` / ``std::basic_string<T>`` API, with a few variations: - -- :cpp:type:`pw::InlineString` provides overloads specific to character arrays. - These perform compile-time capacity checks and are used for class template - argument deduction. Like ``std::string``, character arrays are treated as - null-terminated strings. -- :cpp:type:`pw::InlineString` allows implicit conversions from - ``std::string_view``. Specifying the capacity parameter is cumbersome, so - implicit conversions are helpful. Also, implicitly creating a - :cpp:type:`pw::InlineString` is less costly than creating a ``std::string``. - As with ``std::string``, explicit conversions are required from types that - convert to ``std::string_view``. -- Functions related to dynamic memory allocation are not present (``reserve()``, - ``shrink_to_fit()``, ``get_allocator()``). -- ``resize_and_overwrite()`` only takes the ``Operation`` argument, since the - underlying string buffer cannot be resized. - -See the `std::string documentation -<https://en.cppreference.com/w/cpp/string/basic_string>`_ for full details. - -Key differences from ``std::string`` ------------------------------------- -- **Fixed capacity** -- Operations that add characters to the string beyond its +``pw_string`` provides string classes and utility functions designed to +prioritize safety and static allocation. The APIs are broadly similar to those +of the string classes in the C++ standard library, so familiarity with those +classes will provide some context around ``pw_string`` design decisions. + +------------ +InlineString +------------ +:cpp:type:`pw::InlineString` / :cpp:class:`pw::InlineBasicString` are designed +to match the ``std::string`` / ``std::basic_string<T>`` API as closely as +possible, but with key differences to improve performance on embedded systems: + +- **Fixed capacity:** Operations that add characters to the string beyond its capacity are an error. These trigger a ``PW_ASSERT`` at runtime. When detectable, these situations trigger a ``static_assert`` at compile time. -- **Minimal overhead** -- :cpp:type:`pw::InlineString` operations never +- **Minimal overhead:** :cpp:type:`pw::InlineString` operations never allocate. Reading the contents of the string is a direct memory access within the string object, without pointer indirection. -- **Constexpr support** -- :cpp:type:`pw::InlineString` works in ``constexpr`` +- **Constexpr support:** :cpp:type:`pw::InlineString` works in ``constexpr`` contexts, which is not supported by ``std::string`` until C++20. -Safe Length Checking --------------------- +We don't aim to provide complete API compatibility with +``std::string`` / ``std::basic_string<T>``. Some areas of deviation include: + +- **Compile-time capacity checks:** :cpp:type:`InlineString` provides overloads + specific to character arrays. These perform compile-time capacity checks and + are used for class template argument deduction. +- **Implicit conversions from** ``std::string_view`` **:** Specifying the + capacity parameter is cumbersome, so implicit conversions are helpful. Also, + implicitly creating a :cpp:type:`InlineString` is less costly than creating a + ``std::string``. As with ``std::string``, explicit conversions are required + from types that convert to ``std::string_view``. +- **No dynamic allocation functions:** Functions that allocate memory, like + ``reserve()``, ``shrink_to_fit()``, and ``get_allocator()``, are simply not + present. + +Capacity +======== +:cpp:type:`InlineBasicString` has a template parameter for the capacity, but the +capacity does not need to be known by the user to use the string safely. The +:cpp:type:`InlineBasicString` template inherits from a +:cpp:type:`InlineBasicString` specialization with capacity of the reserved value +``pw::InlineString<>::npos``. The actual capacity is stored in a single word +alongside the size. This allows code to work with strings of any capacity +through a ``InlineString<>`` or ``InlineBasicString<T>`` reference. + +Exceeding the capacity +---------------------- +Any :cpp:type:`pw::InlineString` operations that exceed the string's capacity +fail an assertion, resulting in a crash. Helpers are provided in +``pw_string/util.h`` that return ``pw::Status::ResourceExhausted()`` instead of +failing an assert when the capacity would be exceeded. + +------------------------ +String utility functions +------------------------ + +Safe length checking +==================== This module provides two safer alternatives to ``std::strlen`` in case the string is extremely long and/or potentially not null-terminated. @@ -53,10 +73,3 @@ null-termination. Second, a constexpr specialized form is offered where null termination is required through :cpp:func:`pw::string::NullTerminatedLength`. This will only return a length if the string is null-terminated. - -Exceeding the capacity ----------------------- -Any :cpp:type:`pw::InlineString` operations that exceed the string's capacity -fail an assertion, resulting in a crash. Helpers are provided in -``pw_string/util.h`` that return ``pw::Status::ResourceExhausted()`` instead of -failing an assert when the capacity would be exceeded. diff --git a/pw_string/docs.rst b/pw_string/docs.rst index 7aa800044..e7b46608d 100644 --- a/pw_string/docs.rst +++ b/pw_string/docs.rst @@ -1,108 +1,117 @@ .. _module-pw_string: +.. rst-class:: with-subtitle + ========= pw_string ========= -.. card:: - - :octicon:`comment-discussion` Status: - :bdg-secondary-line:`Experimental` - :octicon:`chevron-right` - :bdg-secondary-line:`Unstable` - :octicon:`chevron-right` - :bdg-primary:`Stable` - :octicon:`kebab-horizontal` - :bdg-primary:`Current` - :octicon:`chevron-right` - :bdg-secondary-line:`Deprecated` - -Compatibility: C++17 (C++14 for :cpp:type:`pw::InlineString`) -`API reference </pw_string/api.html>`_ | `Guide </pw_string/guide.html>`_ | `Design </pw_string/design.html>`_ +.. pigweed-module:: + :name: pw_string + :tagline: Efficient, easy, and safe string manipulation + :status: stable + :languages: C++14, C++17 + :code-size-impact: 500 to 1500 bytes + :get-started: module-pw_string-get-started + :design: module-pw_string-design + :guides: module-pw_string-guide + :api: module-pw_string-api ---------------------------------------------- -Efficient, easy, and safe string manipulation ---------------------------------------------- -- **Efficient**: No memory allocation, no pointer indirection. -- **Easy**: Use the string API you already know. -- **Safe**: Never worry about buffer overruns or undefined behavior. + - **Efficient**: No memory allocation, no pointer indirection. + - **Easy**: Use the string API you already know. + - **Safe**: Never worry about buffer overruns or undefined behavior. -*Pick three!* If you know how to use ``std::string``, just use -:cpp:type:`pw::InlineString` in the same way: + *Pick three!* If you know how to use ``std::string``, just use + :cpp:type:`pw::InlineString` in the same way: -.. code:: cpp + .. code:: cpp - // Create a string from a C-style char array; storage is pre-allocated! - pw::InlineString<16> my_string = "Literally"; + // Create a string from a C-style char array; storage is pre-allocated! + pw::InlineString<16> my_string = "Literally"; - // We have some space left, so let's add to the string. - my_string.append('?', 3); // "Literally???" + // We have some space left, so let's add to the string. + my_string.append('?', 3); // "Literally???" - // Let's try something evil and extend this past its capacity 😈 - my_string.append('!', 8); - // Foiled by a crash! No mysterious bugs or undefined behavior. + // Let's try something evil and extend this past its capacity 😈 + my_string.append('!', 8); + // Foiled by a crash! No mysterious bugs or undefined behavior. -Need to build up a string? :cpp:type:`pw::StringBuilder` works like -``std::ostringstream``, but with most of the efficiency and memory benefits of -:cpp:type:`pw::InlineString`: + Need to build up a string? :cpp:type:`pw::StringBuilder` works like + ``std::ostringstream``, but with most of the efficiency and memory benefits + of :cpp:type:`pw::InlineString`: -.. code:: cpp + .. code:: cpp - // Create a pw::StringBuilder with a built-in buffer - pw::StringBuffer<32> my_string_builder = "Is it really this easy?"; + // Create a pw::StringBuilder with a built-in buffer + pw::StringBuffer<32> my_string_builder = "Is it really this easy?"; - // Add to it with idiomatic C++ - my_string << " YES!"; + // Add to it with idiomatic C++ + my_string << " YES!"; - // Use it like any other string - PW_LOG_DEBUG("%s", my_string_builder.c_str()); + // Use it like any other string + PW_LOG_DEBUG("%s", my_string_builder.c_str()); - -Check out :ref:`module-pw_string-guide` for more code samples. + Check out :ref:`module-pw_string-guide` for more code samples. ---------- Background ---------- -String manipulation is a very common operation, but the standard C and C++ -string libraries have drawbacks. The C++ functions are easy-to-use and powerful, -but require too much flash and memory for many embedded projects. The C string -functions are lighter weight, but can be difficult to use correctly. Mishandling -of null terminators or buffer sizes can result in serious bugs. +String manipulation on embedded systems can be surprisingly challenging. +C strings are light weight but come with many pitfalls for those who don't know +the standard library deeply. C++ provides string classes that are safe and easy +to use, but they consume way too much code space and are designed to be used +with dynamic memory allocation. + +Embedded systems need string functionality that is both safe and suitable for +resource-constrained platforms. ------------ Our solution ------------ -The ``pw_string`` module provides the flexibility, ease-of-use, and safety of -C++-style string manipulation, but with no dynamic memory allocation and a much -smaller binary size impact. Using ``pw_string`` in place of the standard C -functions eliminates issues related to buffer overflow or missing null -terminators. +``pw_string`` provides safe string handling functionality with an API that +closely matches that of ``std::string``, but without dynamic memory allocation +and with a *much* smaller :ref:`binary size impact <module-pw_string-size-reports>`. --------------- Who this is for --------------- -``pw_string`` is potentially useful for anyone who is working with strings in -C++. +``pw_string`` is useful any time you need to handle strings in embedded C++. +-------------------- Is it right for you? -------------------- +If your project written in C, ``pw_string`` is not a good fit since we don't +currently expose a C API. + +For larger platforms where code space isn't in short supply and dynamic memory +allocation isn't a problem, you may find that ``std::string`` meets your needs. + +.. tip:: + ``pw_string`` works just as well on larger embedded platforms and host + systems. Using ``pw_string`` even when you might get away with ``std:string`` + gives you the flexibility to move to smaller platforms later with much less + rework. + Here are some size reports that may affect whether ``pw_string`` is right for you. +.. _module-pw_string-size-reports: + Size comparison: snprintf versus pw::StringBuilder -------------------------------------------------- -:cpp:type:`pw::StringBuilder` is safe, flexible, and results in much smaller code size than -using ``std::ostringstream``. However, applications sensitive to code size -should use :cpp:type:`pw::StringBuilder` with care. - -The fixed code size cost of :cpp:type:`pw::StringBuilder` is significant, though smaller than -``std::snprintf``. Using :cpp:type:`pw::StringBuilder`'s ``<<`` and ``append`` methods exclusively in -place of ``snprintf`` reduces code size, but ``snprintf`` may be difficult to -avoid. - -The incremental code size cost of :cpp:type:`pw::StringBuilder` is comparable to ``snprintf`` if -errors are handled. Each argument to :cpp:type:`pw::StringBuilder`'s ``<<`` method expands to a -function call, but one or two :cpp:type:`pw::StringBuilder` appends may have a smaller code size +:cpp:type:`pw::StringBuilder` is safe, flexible, and results in much smaller +code size than using ``std::ostringstream``. However, applications sensitive to +code size should use :cpp:type:`pw::StringBuilder` with care. + +The fixed code size cost of :cpp:type:`pw::StringBuilder` is significant, though +smaller than ``std::snprintf``. Using :cpp:type:`pw::StringBuilder`'s ``<<`` and +``append`` methods exclusively in place of ``snprintf`` reduces code size, but +``snprintf`` may be difficult to avoid. + +The incremental code size cost of :cpp:type:`pw::StringBuilder` is comparable to +``snprintf`` if errors are handled. Each argument to +:cpp:type:`pw::StringBuilder`'s ``<<`` method expands to a function call, but +one or two :cpp:type:`pw::StringBuilder` appends may have a smaller code size impact than a single ``snprintf`` call. .. include:: string_builder_size_report @@ -115,6 +124,18 @@ incremental code size cost to using ``pw::string::Format``. .. include:: format_size_report +Roadmap +------- +* StringBuilder's fixed size cost can be dramatically reduced by limiting + support for 64-bit integers. +* Consider integrating with the tokenizer module. + +Compatibility +------------- +C++17, C++14 (:cpp:type:`pw::InlineString`) + +.. _module-pw_string-get-started: + --------------- Getting started --------------- @@ -143,25 +164,15 @@ Zephyr ------ Add ``CONFIG_PIGWEED_STRING=y`` to the Zephyr project's configuration. ---------------------- -Design considerations ---------------------- -``pw_string`` is designed to prioritize safety and static allocation. It matches -the ``std::string`` API as closely as possible, but isn't intended to provide -complete API compatibility. See :ref:`module-pw_string-design` for more -details. - ------- Roadmap ------- -* The fixed size cost of :cpp:type:`pw::StringBuilder` can be dramatically reduced by - limiting support for 64-bit integers. +* The fixed size cost of :cpp:type:`pw::StringBuilder` can be dramatically + reduced by limiting support for 64-bit integers. * ``pw_string`` may be integrated with :ref:`module-pw_tokenizer`. ----------- -Learn more ----------- .. toctree:: + :hidden: :maxdepth: 1 design diff --git a/pw_string/guide.rst b/pw_string/guide.rst index 38916ee52..87ceec03e 100644 --- a/pw_string/guide.rst +++ b/pw_string/guide.rst @@ -1,11 +1,67 @@ .. _module-pw_string-guide: -=============== -pw_string guide -=============== +================ +pw_string: Guide +================ -Building strings with StringBuilder ------------------------------------ +InlineString and StringBuilder? +=============================== +Use :cpp:type:`pw::InlineString` if you need: + +* Compatibility with ``std::string`` +* Storage internal to the object +* A string object to persist in other data structures +* Lower code size overhead + +Use :cpp:class:`pw::StringBuilder` if you need: + +* Compatibility with ``std::ostringstream``, including custom object support +* Storage external to the object +* Non-fatal handling of failed append/format operations +* Tracking of the status of a series of operations +* A temporary stack object to aid string construction +* Medium code size overhead + +An example of when to prefer :cpp:type:`pw::InlineString` is wrapping a +length-delimited string (e.g. ``std::string_view``) for APIs that require null +termination: + +.. code-block:: cpp + + #include <string> + #include "pw_log/log.h" + #include "pw_string/string_builder.h" + + void ProcessName(std::string_view name) { + // %s format strings require null terminated strings, so create one on the + // stack with size up to kMaxNameLen, copy the string view `name` contents + // into it, add a null terminator, and log it. + PW_LOG_DEBUG("The name is %s", + pw::InlineString<kMaxNameLen>(name).c_str()); + } + +An example of when to prefer :cpp:class:`pw::StringBuilder` is when +constructing a string for external use. + +.. code-block:: cpp + + #include "pw_string/string_builder.h" + + pw::Status FlushSensorValueToUart(int32_t sensor_value) { + pw::StringBuffer<42> sb; + sb << "Sensor value: "; + sb << sensor_value; // Formats as int. + FlushCStringToUart(sb.c_str()); + + if (!sb.status().ok) { + format_error_metric.Increment(); // Track overflows. + } + return sb.status(); + } + + +Building strings with pw::StringBuilder +======================================= The following shows basic use of a :cpp:class:`pw::StringBuilder`. .. code-block:: cpp @@ -34,92 +90,94 @@ The following shows basic use of a :cpp:class:`pw::StringBuilder`. return sb.status(); } -Constructing pw::InlineString objects -------------------------------------- +Building strings with pw::InlineString +====================================== :cpp:type:`pw::InlineString` objects must be constructed by specifying a fixed capacity for the string. .. code-block:: c++ - // Initialize from a C string. - pw::InlineString<32> inline_string = "Literally"; - inline_string.append('?', 3); // contains "Literally???" + #include "pw_string/string.h" - // Supports copying into known-capacity strings. - pw::InlineString<64> other = inline_string; + // Initialize from a C string. + pw::InlineString<32> inline_string = "Literally"; + inline_string.append('?', 3); // contains "Literally???" - // Supports various helpful std::string functions - if (inline_string.starts_with("Lit") || inline_string == "not\0literally"sv) { - other += inline_string; - } + // Supports copying into known-capacity strings. + pw::InlineString<64> other = inline_string; - // Like std::string, InlineString is always null terminated when accessed - // through c_str(). InlineString can be used to null-terminate - // length-delimited strings for APIs that expect null-terminated strings. - std::string_view file(".gif"); - if (std::fopen(pw::InlineString<kMaxNameLen>(file).c_str(), "r") == nullptr) { - return; - } + // Supports various helpful std::string functions + if (inline_string.starts_with("Lit") || inline_string == "not\0literally"sv) { + other += inline_string; + } - // pw::InlineString integrates well with std::string_view. It supports - // implicit conversions to and from std::string_view. - inline_string = std::string_view("not\0literally", 12); + // Like std::string, InlineString is always null terminated when accessed + // through c_str(). InlineString can be used to null-terminate + // length-delimited strings for APIs that expect null-terminated strings. + std::string_view file(".gif"); + if (std::fopen(pw::InlineString<kMaxNameLen>(file).c_str(), "r") == nullptr) { + return; + } - FunctionThatTakesAStringView(inline_string); + // pw::InlineString integrates well with std::string_view. It supports + // implicit conversions to and from std::string_view. + inline_string = std::string_view("not\0literally", 12); - FunctionThatTakesAnInlineString(std::string_view("1234", 4)); + FunctionThatTakesAStringView(inline_string); -Choosing between InlineString and StringBuilder ------------------------------------------------ -:cpp:type:`pw::InlineString` is comparable to ``std::string``, while -:cpp:class:`pw::StringBuilder` is comparable to ``std::ostringstream``. -Because :cpp:class:`pw::StringBuilder` provides high-level stream functionality, -it has more overhead than :cpp:type:`pw::InlineString`. + FunctionThatTakesAnInlineString(std::string_view("1234", 4)); -Use :cpp:type:`pw::InlineString` unless :cpp:class:`pw::StringBuilder`'s -capabilities are needed. Features unique to :cpp:class:`pw::StringBuilder` -include: +Building strings inside InlineString with a StringBuilder +========================================================= +:cpp:class:`pw::StringBuilder` can build a string in a +:cpp:type:`pw::InlineString`: -* Polymorphic C++ stream-style output, potentially supporting custom types. -* Non-fatal handling of failed append/format operations. -* Tracking the status of a series of operations. -* Building a string in an external buffer. +.. code-block:: c++ -If those features are not required, use :cpp:type:`pw::InlineString`. A common -example of when to prefer :cpp:type:`pw::InlineString` is wrapping a -length-delimited string (e.g. ``std::string_view``) for APIs that require null -termination. + #include "pw_string/string.h" -.. code-block:: cpp + void DoFoo() { + InlineString<32> inline_str; + StringBuilder sb(inline_str); + sb << 123 << "456"; + // inline_str contains "456" + } - void ProcessName(std::string_view name) { - PW_LOG_DEBUG("The name is %s", pw::InlineString<kMaxNameLen>(name).c_str()); +Passing InlineStrings as parameters +=================================== +:cpp:type:`pw::InlineString` objects can be passed to non-templated functions +via type erasure. This saves code size in most cases, since it avoids template +expansions triggered by string size differences. -Operating on unknown size strings ---------------------------------- -All :cpp:type:`pw::InlineString` operations may be performed on strings without -specifying their capacity. +Unknown size strings +-------------------- +To operate on :cpp:type:`pw::InlineString` objects without knowing their type, +use the ``pw::InlineString<>`` type, shown in the examples below: .. code-block:: c++ - void RemoveSuffix(pw::InlineString<>& string, std::string_view suffix) { - if (string.ends_with(suffix)) { - string.resize(string.size() - suffix.size()); - } - } + // Note that the first argument is a generically-sized InlineString. + void RemoveSuffix(pw::InlineString<>& string, std::string_view suffix) { + if (string.ends_with(suffix)) { + string.resize(string.size() - suffix.size()); + } + } - void DoStuff() { - pw::InlineString<32> str1 = "Good morning!"; - RemoveSuffix(str1, " morning!"); + void DoStuff() { + pw::InlineString<32> str1 = "Good morning!"; + RemoveSuffix(str1, " morning!"); - pw::InlineString<40> str2 = "Good"; - RemoveSuffix(str2, " morning!"); + pw::InlineString<40> str2 = "Good"; + RemoveSuffix(str2, " morning!"); - PW_ASSERT(str1 == str2); - } + PW_ASSERT(str1 == str2); + } + +However, generically sized :cpp:type:`pw::InlineString` objects don't work in +``constexpr`` contexts. -Operating on known-size strings -------------------------------- +Known size strings +------------------ :cpp:type:`pw::InlineString` operations on known-size strings may be used in ``constexpr`` expressions. @@ -135,13 +193,8 @@ Operating on known-size strings return string; }(); -Building strings ----------------- -:cpp:class:`pw::StringBuilder` may be used to build a string in a -:cpp:type:`pw::InlineString`. - -Deducing class template arguments with pw::InlineBasicString ------------------------------------------------------------- +Compact initialization of InlineStrings +======================================= :cpp:type:`pw::InlineBasicString` supports class template argument deduction (CTAD) in C++17 and newer. Since :cpp:type:`pw::InlineString` is an alias, CTAD is not supported until C++20. @@ -155,9 +208,8 @@ is not supported until C++20. // In C++20, CTAD may be used with the pw::InlineString alias. pw::InlineString my_other_string("123456789"); - -Printing custom types ---------------------- +Supporting custom types with StringBuilder +========================================== As with ``std::ostream``, StringBuilder supports printing custom types by overriding the ``<<`` operator. This is is done by defining ``operator<<`` in the same namespace as the custom type. For example: diff --git a/pw_string/public/pw_string/format.h b/pw_string/public/pw_string/format.h index 7d05fe0c2..205210a8c 100644 --- a/pw_string/public/pw_string/format.h +++ b/pw_string/public/pw_string/format.h @@ -29,22 +29,24 @@ namespace pw::string { -// Writes a printf-style formatted string to the provided buffer, similarly to -// std::snprintf. Returns the number of characters written, excluding the null -// terminator. The buffer is always null-terminated unless it is empty. -// -// The status is -// -// OkStatus() if the operation succeeded, -// Status::ResourceExhausted() if the buffer was too small to fit the output, -// Status::InvalidArgument() if there was a formatting error. -// +/// @brief Writes a printf-style formatted string to the provided buffer, +/// similarly to `std::snprintf()`. +/// +/// The `std::snprintf()` return value is awkward to interpret, and +/// misinterpreting it can lead to serious bugs. +/// +/// @returns The number of characters written, excluding the null +/// terminator. The buffer is always null-terminated unless it is empty. +/// The status is `OkStatus()` if the operation succeeded, +/// `Status::ResourceExhausted()` if the buffer was too small to fit the output, +/// or `Status::InvalidArgument()` if there was a formatting error. PW_PRINTF_FORMAT(2, 3) StatusWithSize Format(span<char> buffer, const char* format, ...); -// Writes a printf-style formatted string with va_list-packed arguments to the -// provided buffer, similarly to std::vsnprintf. The return value is the same as -// above. +/// @brief Writes a printf-style formatted string with va_list-packed arguments +/// to the provided buffer, similarly to `std::vsnprintf()`. +/// +/// @returns See `pw::string::Format()`. PW_PRINTF_FORMAT(2, 0) StatusWithSize FormatVaList(span<char> buffer, const char* format, diff --git a/pw_string/public/pw_string/string.h b/pw_string/public/pw_string/string.h index 441b218d1..a6041f52a 100644 --- a/pw_string/public/pw_string/string.h +++ b/pw_string/public/pw_string/string.h @@ -13,6 +13,11 @@ // the License. #pragma once +/// @file pw_string/string.h +/// +/// @brief `pw::InlineBasicString` and `pw::InlineString` are safer alternatives +/// to `std::basic_string` and `std::string`. + #include <cstddef> #include <initializer_list> #include <iterator> @@ -36,21 +41,24 @@ namespace pw { -// pw::InlineBasicString<T, kCapacity> is a fixed-capacity version of -// std::basic_string. It implements mostly the same API as std::basic_string, -// but the capacity of the string is fixed at construction and cannot grow. -// Attempting to increase the size beyond the capacity triggers an assert. -// -// A pw::InlineString alias of pw::InlineBasicString<char>, equivalent to -// std::string, is defined below. -// -// pw::InlineBasicString has a template parameter for the capacity, but the -// capacity does not have to be known to use the string. The -// pw::InlineBasicString template inherits from a pw::InlineBasicString -// specialization with capacity of the reserved value pw::InlineString<>::npos. -// The actual capacity is stored in a single word alongside the size. This -// allows code to work with strings of any capacity through a pw::InlineString<> -// or pw::InlineBasicString<T> reference. +/// @brief `pw::InlineBasicString` is a fixed-capacity version of +/// `std::basic_string`. In brief: +/// +/// - It is C++14-compatible and null-terminated. +/// - It stores the string contents inline and uses no dynamic memory. +/// - It implements mostly the same API as `std::basic_string`, but the capacity +/// of the string is fixed at construction and cannot grow. Attempting to +/// increase the size beyond the capacity triggers an assert. +/// +/// `pw::InlineBasicString` is efficient and compact. The current size and +/// capacity are stored in a single word. Accessing its contents is a simple +/// array access within the object, with no pointer indirection, even when +/// working from a generic reference `pw::InlineBasicString<T>` where the +/// capacity is not specified as a template argument. A string object can be +/// used safely without the need to know its capacity. +/// +/// See also `pw::InlineString`, which is an alias of +/// `pw::InlineBasicString<char>` and is equivalent to `std::string`. template <typename T, string_impl::size_type kCapacity = string_impl::kGeneric> class InlineBasicString final : public InlineBasicString<T, string_impl::kGeneric> { @@ -566,6 +574,9 @@ constexpr bool operator>=(const T* lhs, // TODO(b/239996007): Implement other comparison operator overloads. // Aliases + +/// @brief `pw::InlineString` is an alias of `pw::InlineBasicString<char>` and +/// is equivalent to `std::string`. template <string_impl::size_type kCapacity = string_impl::kGeneric> using InlineString = InlineBasicString<char, kCapacity>; diff --git a/pw_string/public/pw_string/string_builder.h b/pw_string/public/pw_string/string_builder.h index 886aa609f..c3c9d1760 100644 --- a/pw_string/public/pw_string/string_builder.h +++ b/pw_string/public/pw_string/string_builder.h @@ -12,6 +12,11 @@ // License for the specific language governing permissions and limitations under // the License. #pragma once +/// @file pw_string/string_builder.h +/// +/// @brief `pw::StringBuilder` facilitates creating formatted strings in a +/// fixed-sized buffer or in a `pw::InlineString`. It is designed to give the +/// flexibility of std::ostringstream, but with a small footprint. #include <algorithm> #include <cstdarg> @@ -32,19 +37,18 @@ namespace pw { /// @class StringBuilder /// -/// `StringBuilder` facilitates building formatted strings in a fixed-size -/// buffer. StringBuilders are always null terminated (unless they are +/// `pw::StringBuilder` instances are always null-terminated (unless they are /// constructed with an empty buffer) and never overflow. Status is tracked for /// each operation and an overall status is maintained, which reflects the most /// recent error. /// -/// A `StringBuilder` does not own the buffer it writes to. It can be used to -/// write strings to any buffer. The StringBuffer template class, defined below, -/// allocates a buffer alongside a `StringBuilder`. +/// `pw::StringBuilder` does not own the buffer it writes to. It can be used +/// to write strings to any buffer. The `pw::StringBuffer` template class, +/// defined below, allocates a buffer alongside a `pw::StringBuilder`. /// -/// `StringBuilder` supports C++-style << output, similar to -/// `std::ostringstream`. It also supports std::string-like append functions and -/// printf-style output. +/// `pw::StringBuilder` supports C++-style `<<` output, similar to +/// `std::ostringstream`. It also supports append functions like `std::string` +/// and `printf`-style output. /// /// Support for custom types is added by overloading `operator<<` in the same /// namespace as the custom type. For example: @@ -65,8 +69,8 @@ namespace pw { /// } // namespace my_project /// @endcode /// -/// The ToString template function can be specialized to support custom types -/// with `StringBuilder`, though overloading `operator<<` is generally +/// The `ToString` template function can be specialized to support custom types +/// with `pw::StringBuilder`, though overloading `operator<<` is generally /// preferred. For example: /// /// @code @@ -82,7 +86,7 @@ namespace pw { /// class StringBuilder { public: - /// Creates an empty StringBuilder. + /// Creates an empty `pw::StringBuilder`. explicit constexpr StringBuilder(span<char> buffer) : buffer_(buffer), size_(&inline_size_), inline_size_(0) { NullTerminate(); @@ -98,7 +102,7 @@ class StringBuilder { inline_size_(0) {} /// Disallow copy/assign to avoid confusion about where the string is actually - /// stored. StringBuffers may be copied into one another. + /// stored. `pw::StringBuffer` instances may be copied into one another. StringBuilder(const StringBuilder&) = delete; StringBuilder& operator=(const StringBuilder&) = delete; @@ -110,40 +114,41 @@ class StringBuilder { const char* data() const { return buffer_.data(); } const char* c_str() const { return data(); } - /// Returns a std::string_view of the contents of this StringBuilder. The - /// std::string_view is invalidated if the StringBuilder contents change. + /// Returns a `std::string_view` of the contents of this `pw::StringBuilder`. + /// The `std::string_view` is invalidated if the `pw::StringBuilder` contents + /// change. std::string_view view() const { return std::string_view(data(), size()); } - /// Allow implicit conversions to std::string_view so StringBuilders can be - /// passed into functions that take a std::string_view. + /// Allow implicit conversions to `std::string_view` so `pw::StringBuilder` + /// instances can be passed into functions that take a `std::string_view`. operator std::string_view() const { return view(); } - /// Returns a span<const std::byte> representation of this StringBuffer. + /// Returns a `span<const std::byte>` representation of this + /// `pw::StringBuffer`. span<const std::byte> as_bytes() const { return span(reinterpret_cast<const std::byte*>(buffer_.data()), size()); } - /// Returns the StringBuilder's status, which reflects the most recent error - /// that occurred while updating the string. After an update fails, the status - /// remains non-OK until it is cleared with clear() or clear_status(). - /// Returns: - /// - /// OK if no errors have occurred - /// RESOURCE_EXHAUSTED if output to the StringBuilder was truncated - /// INVALID_ARGUMENT if printf-style formatting failed - /// OUT_OF_RANGE if an operation outside the buffer was attempted + /// Returns the status of `pw::StringBuilder`, which reflects the most recent + /// error that occurred while updating the string. After an update fails, the + /// status remains non-OK until it is cleared with + /// `pw::StringBuilder::clear()` or `pw::StringBuilder::clear_status()`. /// + /// @returns `OK` if no errors have occurred; `RESOURCE_EXHAUSTED` if output + /// to the `StringBuilder` was truncated; `INVALID_ARGUMENT` if `printf`-style + /// formatting failed; `OUT_OF_RANGE` if an operation outside the buffer was + /// attempted. Status status() const { return static_cast<Status::Code>(status_); } - /// Returns status() and size() as a StatusWithSize. + /// Returns `status()` and `size()` as a `StatusWithSize`. StatusWithSize status_with_size() const { return StatusWithSize(status(), size()); } - /// The status from the last operation. May be OK while status() is not OK. + /// The status from the last operation. May be OK while `status()` is not OK. Status last_status() const { return static_cast<Status::Code>(last_status_); } - /// True if status() is OkStatus(). + /// True if `status()` is `OkStatus()`. bool ok() const { return status().ok(); } /// True if the string is empty. @@ -158,57 +163,58 @@ class StringBuilder { /// Clears the string and resets its error state. void clear(); - /// Sets the statuses to OkStatus(); + /// Sets the statuses to `OkStatus()`; void clear_status() { status_ = static_cast<unsigned char>(OkStatus().code()); last_status_ = static_cast<unsigned char>(OkStatus().code()); } - /// Appends a single character. Stets the status to RESOURCE_EXHAUSTED if the + /// Appends a single character. Sets the status to `RESOURCE_EXHAUSTED` if the /// character cannot be added because the buffer is full. void push_back(char ch) { append(1, ch); } - /// Removes the last character. Sets the status to OUT_OF_RANGE if the buffer - /// is empty (in which case the unsigned overflow is intentional). + /// Removes the last character. Sets the status to `OUT_OF_RANGE` if the + /// buffer is empty (in which case the unsigned overflow is intentional). void pop_back() PW_NO_SANITIZE("unsigned-integer-overflow") { resize(size() - 1); } - /// Appends the provided character count times. + /// Appends the provided character `count` times. StringBuilder& append(size_t count, char ch); - /// Appends count characters from str to the end of the StringBuilder. If - /// count exceeds the remaining space in the StringBuffer, max_size() - size() - /// characters are appended and the status is set to RESOURCE_EXHAUSTED. + /// Appends `count` characters from `str` to the end of the `StringBuilder`. + /// If count exceeds the remaining space in the `StringBuffer`, + /// `max_size() - size()` characters are appended and the status is set to + /// `RESOURCE_EXHAUSTED`. /// - /// str is not considered null-terminated and may contain null characters. + /// `str` is not considered null-terminated and may contain null characters. StringBuilder& append(const char* str, size_t count); /// Appends characters from the null-terminated string to the end of the - /// StringBuilder. If the string's length exceeds the remaining space in the - /// buffer, max_size() - size() characters are copied and the status is set to - /// RESOURCE_EXHAUSTED. + /// `StringBuilder`. If the string's length exceeds the remaining space in the + /// buffer, `max_size() - size()` characters are copied and the status is + /// set to `RESOURCE_EXHAUSTED`. /// - /// This function uses string::Length instead of std::strlen to avoid - /// unbounded reads if the string is not null terminated. + /// This function uses `string::Length` instead of `std::strlen` to avoid + /// unbounded reads if the string is not null-terminated. StringBuilder& append(const char* str); - /// Appends a std::string_view to the end of the StringBuilder. + /// Appends a `std::string_view` to the end of the `StringBuilder`. StringBuilder& append(const std::string_view& str); - /// Appends a substring from the std::string_view to the StringBuilder. Copies - /// up to count characters starting from pos to the end of the StringBuilder. - /// If pos > str.size(), sets the status to OUT_OF_RANGE. + /// Appends a substring from the `std::string_view` to the `StringBuilder`. + /// Copies up to count characters starting from `pos` to the end of the + /// `StringBuilder`. If `pos > str.size()`, sets the status to `OUT_OF_RANGE`. StringBuilder& append(const std::string_view& str, size_t pos, size_t count = std::string_view::npos); - /// Appends to the end of the StringBuilder using the << operator. This - /// enables C++ stream-style formatted to StringBuilders. + /// Appends to the end of the `StringBuilder` using the `<<` operator. This + /// enables C++ stream-style formatted to `StringBuilder` instances. template <typename T> StringBuilder& operator<<(const T& value) { - /// For std::string_view-compatible types, use the append function, which - /// gives smaller code size. + /// For types compatible with `std::string_view`, use the `append` function, + /// which gives smaller code size. if constexpr (std::is_convertible_v<T, std::string_view>) { append(value); } else if constexpr (std::is_convertible_v<T, span<const std::byte>>) { @@ -219,7 +225,7 @@ class StringBuilder { return *this; } - /// Provide a few additional operator<< overloads that reduce code size. + /// Provide a few additional `operator<<` overloads that reduce code size. StringBuilder& operator<<(bool value) { return append(value ? "true" : "false"); } @@ -236,32 +242,32 @@ class StringBuilder { StringBuilder& operator<<(Status status) { return *this << status.str(); } /// @fn pw::StringBuilder::Format - /// Appends a printf-style string to the end of the StringBuilder. If the + /// Appends a `printf`-style string to the end of the `StringBuilder`. If the /// formatted string does not fit, the results are truncated and the status is - /// set to RESOURCE_EXHAUSTED. + /// set to `RESOURCE_EXHAUSTED`. /// /// @param format The format string /// @param ... Arguments for format specification /// - /// @return StringBuilder& + /// @returns `StringBuilder&` /// - /// @note Internally, calls string::Format, which calls std::vsnprintf. + /// @note Internally, calls `string::Format`, which calls `std::vsnprintf`. PW_PRINTF_FORMAT(2, 3) StringBuilder& Format(const char* format, ...); - /// Appends a vsnprintf-style string with va_list arguments to the end of the - /// StringBuilder. If the formatted string does not fit, the results are - /// truncated and the status is set to RESOURCE_EXHAUSTED. + /// Appends a `vsnprintf`-style string with `va_list` arguments to the end of + /// the `StringBuilder`. If the formatted string does not fit, the results are + /// truncated and the status is set to `RESOURCE_EXHAUSTED`. /// - /// Internally, calls string::Format, which calls std::vsnprintf. + /// @note Internally, calls `string::Format`, which calls `std::vsnprintf`. PW_PRINTF_FORMAT(2, 0) StringBuilder& FormatVaList(const char* format, va_list args); - /// Sets the StringBuilder's size. This function only truncates; if - /// new_size > size(), it sets status to OUT_OF_RANGE and does nothing. + /// Sets the size of the `StringBuilder`. This function only truncates; if + /// `new_size > size()`, it sets status to `OUT_OF_RANGE` and does nothing. void resize(size_t new_size); protected: - /// Functions to support StringBuffer copies. + /// Functions to support `StringBuffer` copies. constexpr StringBuilder(span<char> buffer, const StringBuilder& other) : buffer_(buffer), size_(&inline_size_), @@ -272,7 +278,7 @@ class StringBuilder { void CopySizeAndStatus(const StringBuilder& other); private: - /// Statuses are stored as an unsigned char so they pack into a single word. + /// Statuses are stored as an `unsigned char` so they pack into a single word. static constexpr unsigned char StatusCode(Status status) { return static_cast<unsigned char>(status.code()); } @@ -295,16 +301,16 @@ class StringBuilder { InlineString<>::size_type* size_; - /// Place the inline_size_, status_, and last_status_ members together and use - /// unsigned char for the status codes so these members can be packed into a - /// single word. + // Place the `inline_size_`, `status_`, and `last_status_` members together + // and use `unsigned char` for the status codes so these members can be + // packed into a single word. InlineString<>::size_type inline_size_; unsigned char status_ = StatusCode(OkStatus()); unsigned char last_status_ = StatusCode(OkStatus()); }; -// StringBuffers declare a buffer along with a StringBuilder. StringBuffer can -// be used as a statically allocated replacement for std::ostringstream or +// StringBuffer declares a buffer along with a StringBuilder. StringBuffer +// can be used as a statically allocated replacement for std::ostringstream or // std::string. For example: // // StringBuffer<32> str; diff --git a/pw_string/public/pw_string/util.h b/pw_string/public/pw_string/util.h index 2e06b6e9f..d95bb798f 100644 --- a/pw_string/public/pw_string/util.h +++ b/pw_string/public/pw_string/util.h @@ -12,6 +12,10 @@ // License for the specific language governing permissions and limitations under // the License. #pragma once +/// @file pw_string/util.h +/// +/// @brief The `pw::string::*` functions provide safer alternatives to +/// C++ standard library string functions. #include <cctype> #include <cstddef> @@ -46,11 +50,11 @@ PW_CONSTEXPR_CPP20 inline StatusWithSize CopyToSpan( } // namespace internal -// Safe alternative to the string_view constructor to avoid the risk of an -// unbounded implicit or explicit use of strlen. -// -// This is strongly recommended over using something like C11's strnlen_s as -// a string_view does not require null-termination. +/// @brief Safe alternative to the `string_view` constructor that avoids the +/// risk of an unbounded implicit or explicit use of `strlen`. +/// +/// This is strongly recommended over using something like C11's `strnlen_s` as +/// a `string_view` does not require null-termination. constexpr std::string_view ClampedCString(span<const char> str) { return std::string_view(str.data(), internal::ClampedLength(str.data(), str.size())); @@ -60,15 +64,16 @@ constexpr std::string_view ClampedCString(const char* str, size_t max_len) { return ClampedCString(span<const char>(str, max_len)); } -// Safe alternative to strlen to calculate the null-terminated length of the -// string within the specified span, excluding the null terminator. Like C11's -// strnlen_s, the scan for the null-terminator is bounded. -// -// Returns: -// null-terminated length of the string excluding the null terminator. -// OutOfRange - if the string is not null-terminated. -// -// Precondition: The string shall be at a valid pointer. +/// @brief `pw::string::NullTerminatedLength` is a safer alternative to +/// `strlen` for calculating the null-terminated length of the +/// string within the specified span, excluding the null terminator. +/// +/// Like `strnlen_s` in C11, the scan for the null-terminator is bounded. +/// +/// @pre The string shall be at a valid pointer. +/// +/// @returns the null-terminated length of the string excluding the null +/// terminator or `OutOfRange` if the string is not null-terminated. constexpr Result<size_t> NullTerminatedLength(span<const char> str) { PW_DASSERT(str.data() != nullptr); @@ -84,14 +89,17 @@ constexpr Result<size_t> NullTerminatedLength(const char* str, size_t max_len) { return NullTerminatedLength(span<const char>(str, max_len)); } -// Copies the source string to the dest, truncating if the full string does not -// fit. Always null terminates if dest.size() or num > 0. -// -// Returns the number of characters written, excluding the null terminator. If -// the string is truncated, the status is ResourceExhausted. -// -// Precondition: The destination and source shall not overlap. -// Precondition: The source shall be a valid pointer. +/// @brief `pw::string::Copy` is a safer alternative to `std::strncpy` as it +/// always null-terminates whenever the destination buffer has a non-zero size. +/// +/// Copies the `source` string to the `dest`, truncating if the full string does +/// not fit. Always null terminates if `dest.size()` or `num` is greater than 0. +/// +/// @pre The destination and source shall not overlap. The source +/// shall be a valid pointer. +/// +/// @returns the number of characters written, excluding the null terminator. If +/// the string is truncated, the status is `RESOURCE_EXHAUSTED`. template <typename Span> PW_CONSTEXPR_CPP20 inline StatusWithSize Copy(const std::string_view& source, Span&& dest) { @@ -116,13 +124,14 @@ PW_CONSTEXPR_CPP20 inline StatusWithSize Copy(const char* source, return Copy(source, span<char>(dest, num)); } -// Assigns a std::string_view to a pw::InlineString, truncating if it does not -// fit. pw::InlineString's assign() function asserts if the string's requested -// size exceeds its capacity; pw::string::Assign() returns a Status instead. -// -// Returns: -// OK - the entire std::string_view was copied to the end of the InlineString -// RESOURCE_EXHAUSTED - the std::string_view was truncated to fit +/// Assigns a `std::string_view` to a `pw::InlineString`, truncating if it does +/// not fit. The `assign()` function of `pw::InlineString` asserts if the +/// string's requested size exceeds its capacity; `pw::string::Assign()` +/// returns a `Status` instead. +/// +/// @return `OK` if the entire `std::string_view` was copied to the end of the +/// `pw::InlineString`. `RESOURCE_EXHAUSTED` if the `std::string_view` was +/// truncated to fit. inline Status Assign(InlineString<>& string, const std::string_view& view) { const size_t chars_copied = std::min(view.size(), static_cast<size_t>(string.capacity())); @@ -136,13 +145,13 @@ inline Status Assign(InlineString<>& string, const char* c_string) { return Assign(string, ClampedCString(c_string, string.capacity() + 1)); } -// Appends a std::string_view to a pw::InlineString, truncating if it does not -// fit. pw::InlineString's append() function asserts if the string's requested -// size exceeds its capacity; pw::string::Append() returns a Status instead. -// -// Returns: -// OK - the entire std::string_view was assigned -// RESOURCE_EXHAUSTED - the std::string_view was truncated to fit +/// Appends a `std::string_view` to a `pw::InlineString`, truncating if it +/// does not fit. The `append()` function of `pw::InlineString` asserts if the +/// string's requested size exceeds its capacity; `pw::string::Append()` returns +/// a `Status` instead. +/// +/// @return `OK` if the entire `std::string_view` was assigned. +/// `RESOURCE_EXHAUSTED` if the `std::string_view` was truncated to fit. inline Status Append(InlineString<>& string, const std::string_view& view) { const size_t chars_copied = std::min( view.size(), static_cast<size_t>(string.capacity() - string.size())); @@ -156,8 +165,11 @@ inline Status Append(InlineString<>& string, const char* c_string) { return Append(string, ClampedCString(c_string, string.capacity() + 1)); } -// Copies source string to the dest with same behavior as Copy, with the -// difference that any non-printable characters are changed to '.'. +/// @brief Provides a safe, printable copy of a string. +/// +/// Copies the `source` string to the `dest` string with same behavior as +/// `pw::string::Copy`, with the difference that any non-printable characters +/// are changed to `.`. PW_CONSTEXPR_CPP20 inline StatusWithSize PrintableCopy( const std::string_view& source, span<char> dest) { StatusWithSize copy_result = Copy(source, dest); diff --git a/pw_sync/BUILD.bazel b/pw_sync/BUILD.bazel index 78fc7b2f0..5d48dac9c 100644 --- a/pw_sync/BUILD.bazel +++ b/pw_sync/BUILD.bazel @@ -54,7 +54,6 @@ pw_cc_library( name = "binary_semaphore_backend_multiplexer", visibility = ["@pigweed_config//:__pkg__"], deps = select({ - "@platforms//os:none": ["//pw_sync_baremetal:binary_semaphore"], "//pw_build/constraints/rtos:embos": ["//pw_sync_embos:binary_semaphore"], "//pw_build/constraints/rtos:freertos": ["//pw_sync_freertos:binary_semaphore"], "//pw_build/constraints/rtos:threadx": ["//pw_sync_threadx:binary_semaphore"], @@ -89,7 +88,6 @@ pw_cc_library( name = "counting_semaphore_backend_multiplexer", visibility = ["@pigweed_config//:__pkg__"], deps = select({ - "@platforms//os:none": ["//pw_sync_baremetal:counting_semaphore"], "//pw_build/constraints/rtos:embos": ["//pw_sync_embos:counting_semaphore"], "//pw_build/constraints/rtos:freertos": ["//pw_sync_freertos:counting_semaphore"], "//pw_build/constraints/rtos:threadx": ["//pw_sync_threadx:counting_semaphore"], @@ -175,7 +173,6 @@ pw_cc_library( name = "mutex_backend_multiplexer", visibility = ["@pigweed_config//:__pkg__"], deps = select({ - "@platforms//os:none": ["//pw_sync_baremetal:mutex"], "//pw_build/constraints/rtos:embos": ["//pw_sync_embos:mutex"], "//pw_build/constraints/rtos:freertos": ["//pw_sync_freertos:mutex"], "//pw_build/constraints/rtos:threadx": ["//pw_sync_threadx:mutex"], @@ -215,7 +212,6 @@ pw_cc_library( name = "timed_mutex_backend_multiplexer", visibility = ["@pigweed_config//:__pkg__"], deps = select({ - "@platforms//os:none": ["//pw_sync_baremetal:timed_mutex"], "//pw_build/constraints/rtos:embos": ["//pw_sync_embos:timed_mutex"], "//pw_build/constraints/rtos:freertos": ["//pw_sync_freertos:timed_mutex"], "//pw_build/constraints/rtos:threadx": ["//pw_sync_threadx:timed_mutex"], @@ -280,7 +276,6 @@ pw_cc_library( name = "interrupt_spin_lock_backend_multiplexer", visibility = ["@pigweed_config//:__pkg__"], deps = select({ - "@platforms//os:none": ["//pw_sync_baremetal:interrupt_spin_lock"], "//pw_build/constraints/rtos:embos": ["//pw_sync_embos:interrupt_spin_lock"], "//pw_build/constraints/rtos:freertos": ["//pw_sync_freertos:interrupt_spin_lock"], "//pw_build/constraints/rtos:threadx": ["//pw_sync_threadx:interrupt_spin_lock"], @@ -308,7 +303,8 @@ pw_cc_library( name = "thread_notification_backend_multiplexer", visibility = ["@pigweed_config//:__pkg__"], deps = select({ - "//conditions:default": ["//pw_sync:binary_semaphore_thread_notification_backend"], + "//pw_build/constraints/rtos:freertos": ["//pw_sync_freertos:thread_notification"], + "//conditions:default": [":binary_semaphore_thread_notification_backend"], }), ) @@ -319,7 +315,7 @@ pw_cc_facade( ], includes = ["public"], deps = [ - ":thread_notification_facade", + ":thread_notification", "//pw_chrono:system_clock", ], ) @@ -337,6 +333,7 @@ pw_cc_library( name = "timed_thread_notification_backend_multiplexer", visibility = ["@pigweed_config//:__pkg__"], deps = select({ + "//pw_build/constraints/rtos:freertos": ["//pw_sync_freertos:timed_thread_notification"], "//conditions:default": ["//pw_sync:binary_semaphore_timed_thread_notification_backend"], }), ) diff --git a/pw_sync/docs.rst b/pw_sync/docs.rst index 6e1adb441..e30c01f51 100644 --- a/pw_sync/docs.rst +++ b/pw_sync/docs.rst @@ -33,8 +33,9 @@ The critical section lock primitives provided by this module comply with relevant `TimedLockable <https://en.cppreference.com/w/cpp/named_req/TimedLockable>`_ C++ named requirements. This means that they are compatible with existing helpers in -the STL's ``<mutex>`` thread support library. For example `std::lock_guard <https://en.cppreference.com/w/cpp/thread/lock_guard>`_ -and `std::unique_lock <https://en.cppreference.com/w/cpp/thread/unique_lock>`_ can be directly used. +the STL's ``<mutex>`` thread support library. For example `std::lock_guard +<https://en.cppreference.com/w/cpp/thread/lock_guard>`_ and `std::unique_lock +<https://en.cppreference.com/w/cpp/thread/unique_lock>`_ can be directly used. Mutex ===== @@ -186,21 +187,25 @@ Example in C TimedMutex ========== -The TimedMutex is an extension of the Mutex which offers timeout and deadline -based semantics. +.. cpp:namespace-push:: pw::sync -The TimedMutex's API is C++11 STL +The :cpp:class:`TimedMutex` is an extension of the Mutex which offers timeout +and deadline based semantics. + +The :cpp:class:`TimedMutex`'s API is C++11 STL `std::timed_mutex <https://en.cppreference.com/w/cpp/thread/timed_mutex>`_ like, meaning it is a `BasicLockable <https://en.cppreference.com/w/cpp/named_req/BasicLockable>`_, `Lockable <https://en.cppreference.com/w/cpp/named_req/Lockable>`_, and `TimedLockable <https://en.cppreference.com/w/cpp/named_req/TimedLockable>`_. -Note that the ``TimedMutex`` is a derived ``Mutex`` class, meaning that -a ``TimedMutex`` can be used by someone who needs the basic ``Mutex``. This is -in contrast to the C++ STL's +Note that the :cpp:class:`TimedMutex` is a derived :cpp:class:`Mutex` class, +meaning that a :cpp:class:`TimedMutex` can be used by someone who needs the +basic :cpp:class:`Mutex`. This is in contrast to the C++ STL's `std::timed_mutex <https://en.cppreference.com/w/cpp/thread/timed_mutex>`_. +.. cpp:namespace-pop:: + .. list-table:: :header-rows: 1 @@ -1028,19 +1033,22 @@ efficiently as possible for the platform that it is used on. This simpler but highly portable class of signaling primitives is intended to ensure that a portability efficiency tradeoff does not have to be made up front. Today this is class of simpler signaling primitives is limited to the -``pw::sync::ThreadNotification`` and ``pw::sync::TimedThreadNotification``. +:cpp:class:`pw::sync::ThreadNotification` and +:cpp:class:`pw::sync::TimedThreadNotification`. ThreadNotification ================== -The ThreadNotification is a synchronization primitive that can be used to +.. cpp:namespace-push:: pw::sync + +The :cpp:class:`ThreadNotification` is a synchronization primitive that can be used to permit a SINGLE thread to block and consume a latching, saturating notification from multiple notifiers. .. Note:: - Although only a single thread can block on a ThreadNotification at a time, - many instances may be used by a single thread just like binary semaphores. - This is in contrast to some native RTOS APIs, such as direct task - notifications, which re-use the same state within a thread's context. + Although only a single thread can block on a :cpp:class:`ThreadNotification` + at a time, many instances may be used by a single thread just like binary + semaphores. This is in contrast to some native RTOS APIs, such as direct + task notifications, which re-use the same state within a thread's context. .. Warning:: This is a single consumer/waiter, multiple producer/notifier API! @@ -1048,7 +1056,7 @@ notification from multiple notifiers. result, having multiple threads receiving notifications via the acquire API is unsupported. -This is effectively a subset of the ``pw::sync::BinarySemaphore`` API, except +This is effectively a subset of the :cpp:class:`BinarySemaphore` API, except that only a single thread can be notified and block at a time. The single consumer aspect of the API permits the use of a smaller and/or @@ -1057,13 +1065,17 @@ backed by the most efficient native primitive for a target, regardless of whether that is a semaphore, event flag group, condition variable, or something else. -The ThreadNotification is initialized to being empty (latch is not set). +The :cpp:class:`ThreadNotification` is initialized to being empty (latch is not +set). + +.. cpp:namespace-pop:: Generic BinarySemaphore-based Backend ------------------------------------- -This module provides a generic backend for ``pw::sync::ThreadNotification`` via +This module provides a generic backend for +:cpp:class:`pw::sync::ThreadNotification` via ``pw_sync:binary_semaphore_thread_notification`` which uses a -``pw::sync::BinarySemaphore`` as the backing primitive. See +:cpp:class:`pw::sync::BinarySemaphore` as the backing primitive. See :ref:`BinarySemaphore <module-pw_sync-binary-semaphore>` for backend availability. @@ -1157,10 +1169,12 @@ Examples in C++ TimedThreadNotification ======================= -The TimedThreadNotification is an extension of the ThreadNotification which -offers timeout and deadline based semantics. +The :cpp:class:`TimedThreadNotification` is an extension of the +:cpp:class:`ThreadNotification` which offers timeout and deadline based +semantics. -The TimedThreadNotification is initialized to being empty (latch is not set). +The :cpp:class:`TimedThreadNotification` is initialized to being empty (latch is +not set). .. Warning:: This is a single consumer/waiter, multiple producer/notifier API! The @@ -1170,9 +1184,10 @@ The TimedThreadNotification is initialized to being empty (latch is not set). Generic BinarySemaphore-based Backend ------------------------------------- -This module provides a generic backend for ``pw::sync::TimedThreadNotification`` -via ``pw_sync:binary_semaphore_timed_thread_notification`` which uses a -``pw::sync::BinarySemaphore`` as the backing primitive. See +This module provides a generic backend for +:cpp:class:`pw::sync::TimedThreadNotification` via +``pw_sync:binary_semaphore_timed_thread_notification`` which uses a +:cpp:class:`pw::sync::BinarySemaphore` as the backing primitive. See :ref:`BinarySemaphore <module-pw_sync-binary-semaphore>` for backend availability. @@ -1274,22 +1289,27 @@ Examples in C++ CountingSemaphore ================= -The CountingSemaphore is a synchronization primitive that can be used for -counting events and/or resource management where receiver(s) can block on -acquire until notifier(s) signal by invoking release. +.. cpp:namespace-push:: pw::sync + +The :cpp:class:`CountingSemaphore` is a synchronization primitive that can be +used for counting events and/or resource management where receiver(s) can block +on acquire until notifier(s) signal by invoking release. -Note that unlike Mutexes, priority inheritance is not used by semaphores meaning -semaphores are subject to unbounded priority inversions. Due to this, Pigweed -does not recommend semaphores for mutual exclusion. +Note that unlike :cpp:class:`Mutex`, priority inheritance is not used by +semaphores meaning semaphores are subject to unbounded priority inversions. Due +to this, Pigweed does not recommend semaphores for mutual exclusion. -The CountingSemaphore is initialized to being empty or having no tokens. +The :cpp:class:`CountingSemaphore` is initialized to being empty or having no +tokens. The entire API is thread safe, but only a subset is interrupt safe. .. Note:: If there is only a single consuming thread, we recommend using a - ThreadNotification instead which can be much more efficient on some RTOSes - such as FreeRTOS. + :cpp:class:`ThreadNotification` instead which can be much more efficient on + some RTOSes such as FreeRTOS. + +.. cpp:namespace-pop:: .. Warning:: Releasing multiple tokens is often not natively supported, meaning you may @@ -1407,14 +1427,19 @@ you detect whether you ever fall behind. BinarySemaphore =============== -BinarySemaphore is a specialization of CountingSemaphore with an arbitrary token -limit of 1. Note that that ``max()`` is >= 1, meaning it may be released up to -``max()`` times but only acquired once for those N releases. +.. cpp:namespace-push:: pw::sync -Implementations of BinarySemaphore are typically more efficient than the -default implementation of CountingSemaphore. +:cpp:class:`BinarySemaphore` is a specialization of CountingSemaphore with an +arbitrary token limit of 1. Note that that ``max()`` is >= 1, meaning it may be +released up to ``max()`` times but only acquired once for those N releases. -The BinarySemaphore is initialized to being empty or having no tokens. +Implementations of :cpp:class:`BinarySemaphore` are typically more +efficient than the default implementation of :cpp:class:`CountingSemaphore`. + +The :cpp:class:`BinarySemaphore` is initialized to being empty or having no +tokens. + +.. cpp:namespace-pop:: The entire API is thread safe, but only a subset is interrupt safe. @@ -1523,7 +1548,8 @@ Examples in C++ Conditional Variables ===================== -``pw::sync::ConditionVariable`` provides a condition variable implementation -that provides semantics and an API very similar to `std::condition_variable +:cpp:class:`pw::sync::ConditionVariable` provides a condition variable +implementation that provides semantics and an API very similar to +`std::condition_variable <https://en.cppreference.com/w/cpp/thread/condition_variable>`_ in the C++ Standard Library. diff --git a/pw_sync/public/pw_sync/binary_semaphore.h b/pw_sync/public/pw_sync/binary_semaphore.h index 8ae96a0ac..c3e025dea 100644 --- a/pw_sync/public/pw_sync/binary_semaphore.h +++ b/pw_sync/public/pw_sync/binary_semaphore.h @@ -25,13 +25,11 @@ namespace pw::sync { -/// @class BinarySemaphore -/// -/// BinarySemaphore is a specialization of CountingSemaphore with an arbitrary -/// token limit of 1. Note that that max() is >= 1, meaning it may be -/// released up to max() times but only acquired once for those N releases. -/// Implementations of BinarySemaphore are typically more efficient than the -/// default implementation of CountingSemaphore. The entire API is thread safe +/// `BinarySemaphore` is a specialization of `CountingSemaphore` with an +/// arbitrary token limit of 1. Note that that max() is >= 1, meaning it may be +/// released up to `max()` times but only acquired once for those `N` releases. +/// Implementations of `BinarySemaphore` are typically more efficient than the +/// default implementation of `CountingSemaphore`. The entire API is thread safe /// but only a subset is IRQ safe. /// /// WARNING: In order to support global statically constructed BinarySemaphores, @@ -39,7 +37,7 @@ namespace pw::sync { /// environment is done prior to the creation and/or initialization of the /// native synchronization primitives (e.g. kernel initialization). /// -/// The BinarySemaphore is initialized to being empty or having no tokens. +/// The `BinarySemaphore` is initialized to being empty or having no tokens. class BinarySemaphore { public: using native_handle_type = backend::NativeBinarySemaphoreHandle; diff --git a/pw_sync/public/pw_sync/borrow.h b/pw_sync/public/pw_sync/borrow.h index cdb1c4e61..204b888a9 100644 --- a/pw_sync/public/pw_sync/borrow.h +++ b/pw_sync/public/pw_sync/borrow.h @@ -23,9 +23,7 @@ namespace pw::sync { -/// @class BorrowedPointer -/// -/// The BorrowedPointer is an RAII handle which wraps a pointer to a borrowed +/// The `BorrowedPointer` is an RAII handle which wraps a pointer to a borrowed /// object along with a held lock which is guarding the object. When destroyed, /// the lock is released. template <typename GuardedType, typename Lock = pw::sync::VirtualBasicLockable> @@ -73,7 +71,7 @@ class BorrowedPointer { /// /// @rst /// .. note:: - /// The member of pointer member access operator, operator->(), is + /// The member of pointer member access operator, ``operator->()``, is /// recommended over this API as this is prone to leaking references. /// However, this is sometimes necessary. /// @@ -103,16 +101,14 @@ class BorrowedPointer { GuardedType* object_; }; -/// @class Borrowable -/// -/// The Borrowable is a helper construct that enables callers to borrow an +/// The `Borrowable` is a helper construct that enables callers to borrow an /// object which is guarded by a lock. /// /// Users who need access to the guarded object can ask to acquire a -/// BorrowedPointer which permits access while the lock is held. +/// `BorrowedPointer` which permits access while the lock is held. /// -/// This class is compatible with locks which comply with BasicLockable, -/// Lockable, and TimedLockable C++ named requirements. +/// This class is compatible with locks which comply with `BasicLockable`, +/// `Lockable`, and `TimedLockable` C++ named requirements. template <typename GuardedType, typename Lock = pw::sync::VirtualBasicLockable> class Borrowable { public: @@ -141,7 +137,7 @@ class Borrowable { /// Tries to borrow the object. Blocks until the specified timeout has elapsed /// or the object has been borrowed, whichever comes first. Returns a - /// BorrowedPointer on success, otherwise `std::nullopt` (nothing). + /// `BorrowedPointer` on success, otherwise `std::nullopt` (nothing). template <class Rep, class Period> std::optional<BorrowedPointer<GuardedType, Lock>> try_acquire_for( std::chrono::duration<Rep, Period> timeout) { @@ -153,7 +149,7 @@ class Borrowable { /// Tries to borrow the object. Blocks until the specified deadline has passed /// or the object has been borrowed, whichever comes first. Returns a - /// BorrowedPointer on success, otherwise `std::nullopt` (nothing). + /// `BorrowedPointer` on success, otherwise `std::nullopt` (nothing). template <class Clock, class Duration> std::optional<BorrowedPointer<GuardedType, Lock>> try_acquire_until( std::chrono::time_point<Clock, Duration> deadline) { diff --git a/pw_sync/public/pw_sync/counting_semaphore.h b/pw_sync/public/pw_sync/counting_semaphore.h index ec545561e..b0ab87671 100644 --- a/pw_sync/public/pw_sync/counting_semaphore.h +++ b/pw_sync/public/pw_sync/counting_semaphore.h @@ -25,9 +25,7 @@ namespace pw::sync { -/// @class CountingSemaphore -/// -/// The CountingSemaphore is a synchronization primitive that can be used for +/// The `CountingSemaphore` is a synchronization primitive that can be used for /// counting events and/or resource management where receiver(s) can block on /// acquire until notifier(s) signal by invoking release. /// Note that unlike Mutexes, priority inheritance is not used by semaphores @@ -37,13 +35,13 @@ namespace pw::sync { /// /// @rst /// .. WARNING:: -/// In order to support global statically constructed CountingSemaphores the -/// user and/or backend MUST ensure that any initialization required in your -/// environment is done prior to the creation and/or initialization of the -/// native synchronization primitives (e.g. kernel initialization). -/// -/// The CountingSemaphore is initialized to being empty or having no tokens. +/// In order to support global statically constructed ``CountingSemaphores`` +/// the user and/or backend MUST ensure that any initialization required in +/// your environment is done prior to the creation and/or initialization of +/// the native synchronization primitives (e.g. kernel initialization). /// @endrst +/// +/// The `CountingSemaphore` is initialized to being empty or having no tokens. class CountingSemaphore { public: using native_handle_type = backend::NativeCountingSemaphoreHandle; diff --git a/pw_sync/public/pw_sync/inline_borrowable.h b/pw_sync/public/pw_sync/inline_borrowable.h index 546baf818..0a423f40a 100644 --- a/pw_sync/public/pw_sync/inline_borrowable.h +++ b/pw_sync/public/pw_sync/inline_borrowable.h @@ -23,14 +23,12 @@ namespace pw::sync { -/// @class InlineBorrowable -/// -/// InlineBorrowable holds an object of GuardedType and a Lock that guards +/// `InlineBorrowable` holds an object of `GuardedType` and a Lock that guards /// access to the object. It should be used when an object should be guarded for /// its entire lifecycle by a single lock. /// /// This object should be shared with other componetns as a reference of type -/// Borrowable<GuardedType, LockInterface>. +/// `Borrowable<GuardedType, LockInterface>`. /// template <typename GuardedType, typename Lock = pw::sync::VirtualMutex, diff --git a/pw_sync/public/pw_sync/interrupt_spin_lock.h b/pw_sync/public/pw_sync/interrupt_spin_lock.h index 01f93d834..7726c92a8 100644 --- a/pw_sync/public/pw_sync/interrupt_spin_lock.h +++ b/pw_sync/public/pw_sync/interrupt_spin_lock.h @@ -25,16 +25,14 @@ namespace pw::sync { -/// @class InterruptSpinLock -/// -/// The InterruptSpinLock is a synchronization primitive that can be used to +/// The `InterruptSpinLock` is a synchronization primitive that can be used to /// protect shared data from being simultaneously accessed by multiple threads /// and/or interrupts as a targeted global lock, with the exception of /// Non-Maskable Interrupts (NMIs). /// It offers exclusive, non-recursive ownership semantics where IRQs up to a /// backend defined level of "NMIs" will be masked to solve priority-inversion. /// -/// @note This InterruptSpinLock relies on built-in local interrupt masking to +/// @note This `InterruptSpinLock` relies on built-in local interrupt masking to /// make it interrupt safe without requiring the caller to separately mask /// and unmask interrupts when using this primitive. /// @@ -45,7 +43,7 @@ namespace pw::sync { /// /// This entire API is IRQ safe, but NOT NMI safe. /// -/// @b Precondition: Code that holds a specific InterruptSpinLock must not try +/// @b Precondition: Code that holds a specific `InterruptSpinLock` must not try /// to re-acquire it. However, it is okay to nest distinct spinlocks. class PW_LOCKABLE("pw::sync::InterruptSpinLock") InterruptSpinLock { public: @@ -124,22 +122,16 @@ typedef struct pw_sync_InterruptSpinLock pw_sync_InterruptSpinLock; PW_EXTERN_C_START -/// @fn pw_sync_InterruptSpinLock_Lock -/// /// Invokes the `InterruptSpinLock::lock` member function on the given /// `interrupt_spin_lock`. void pw_sync_InterruptSpinLock_Lock(pw_sync_InterruptSpinLock* spin_lock) PW_NO_LOCK_SAFETY_ANALYSIS; -/// @fn pw_sync_InterruptSpinLock_TryLock -/// /// Invokes the `InterruptSpinLock::try_lock` member function on the given /// `interrupt_spin_lock`. bool pw_sync_InterruptSpinLock_TryLock(pw_sync_InterruptSpinLock* spin_lock) PW_NO_LOCK_SAFETY_ANALYSIS; -/// @fn pw_sync_InterruptSpinLock_Unlock -/// /// Invokes the `InterruptSpinLock::unlock` member function on the given /// `interrupt_spin_lock`. void pw_sync_InterruptSpinLock_Unlock(pw_sync_InterruptSpinLock* spin_lock) diff --git a/pw_sync/public/pw_sync/mutex.h b/pw_sync/public/pw_sync/mutex.h index f1137b0f3..5b4246812 100644 --- a/pw_sync/public/pw_sync/mutex.h +++ b/pw_sync/public/pw_sync/mutex.h @@ -25,13 +25,11 @@ namespace pw::sync { -/// @class Mutex -/// -/// The Mutex is a synchronization primitive that can be used to protect shared -/// data from being simultaneously accessed by multiple threads. It offers -/// exclusive, non-recursive ownership semantics where priority inheritance is -/// used to solve the classic priority-inversion problem. This is thread safe, -/// but NOT IRQ safe. +/// The `Mutex` is a synchronization primitive that can be used to protect +/// shared data from being simultaneously accessed by multiple threads. It +/// offers exclusive, non-recursive ownership semantics where priority +/// inheritance is used to solve the classic priority-inversion problem. This +/// is thread safe, but NOT IRQ safe. /// /// @rst /// .. warning:: @@ -129,18 +127,12 @@ typedef struct pw_sync_Mutex pw_sync_Mutex; PW_EXTERN_C_START -/// @fn pw_sync_Mutex_Lock -/// /// Invokes the `Mutex::lock` member function on the given `mutex`. void pw_sync_Mutex_Lock(pw_sync_Mutex* mutex) PW_NO_LOCK_SAFETY_ANALYSIS; -/// @fn pw_sync_Mutex_TryLock -/// /// Invokes the `Mutex::try_lock` member function on the given `mutex`. bool pw_sync_Mutex_TryLock(pw_sync_Mutex* mutex) PW_NO_LOCK_SAFETY_ANALYSIS; -/// @fn pw_sync_Mutex_Unlock -/// /// Invokes the `Mutex::unlock` member function on the given `mutex`. void pw_sync_Mutex_Unlock(pw_sync_Mutex* mutex) PW_NO_LOCK_SAFETY_ANALYSIS; diff --git a/pw_sync/public/pw_sync/thread_notification.h b/pw_sync/public/pw_sync/thread_notification.h index e82ed774b..85e6e1b6b 100644 --- a/pw_sync/public/pw_sync/thread_notification.h +++ b/pw_sync/public/pw_sync/thread_notification.h @@ -17,9 +17,7 @@ namespace pw::sync { -/// @class ThreadNotification -/// -/// The ThreadNotification is a synchronization primitive that can be used to +/// The `ThreadNotification` is a synchronization primitive that can be used to /// permit a SINGLE thread to block and consume a latching, saturating /// notification from multiple notifiers. /// @@ -34,7 +32,7 @@ namespace pw::sync { /// The single consumer aspect of the API permits the use of a smaller and/or /// faster native APIs such as direct thread signaling. /// -/// The ThreadNotification is initialized to being empty (latch is not set). +/// The `ThreadNotification` is initialized to being empty (latch is not set). class ThreadNotification { public: using native_handle_type = backend::NativeThreadNotificationHandle; diff --git a/pw_sync/public/pw_sync/timed_mutex.h b/pw_sync/public/pw_sync/timed_mutex.h index 0640b3713..abb48c1ee 100644 --- a/pw_sync/public/pw_sync/timed_mutex.h +++ b/pw_sync/public/pw_sync/timed_mutex.h @@ -26,13 +26,11 @@ namespace pw::sync { -/// @class TimedMutex -/// -/// The TimedMutex is a synchronization primitive that can be used to protect +/// The `TimedMutex` is a synchronization primitive that can be used to protect /// shared data from being simultaneously accessed by multiple threads with -/// timeouts and deadlines, extending the Mutex. It offers exclusive, +/// timeouts and deadlines, extending the `Mutex`. It offers exclusive, /// non-recursive ownership semantics where priority inheritance is used to -/// solve the classic priority-inversion problem. This is thread safe, but NOT +/// solve the classic priority-inversion problem. This is thread safe, but NOT /// IRQ safe. /// /// @rst diff --git a/pw_sync/public/pw_sync/timed_thread_notification.h b/pw_sync/public/pw_sync/timed_thread_notification.h index 945c0e0c0..7b98b39d8 100644 --- a/pw_sync/public/pw_sync/timed_thread_notification.h +++ b/pw_sync/public/pw_sync/timed_thread_notification.h @@ -18,10 +18,8 @@ namespace pw::sync { -/// @class TimedThreadNotification -/// -/// The TimedThreadNotification is a synchronization primitive that can be used -/// to permit a SINGLE thread to block and consume a latching, saturating +/// The `TimedThreadNotification` is a synchronization primitive that can be +/// used to permit a SINGLE thread to block and consume a latching, saturating /// notification from multiple notifiers. /// /// @b IMPORTANT: This is a single consumer/waiter, multiple producer/notifier @@ -35,7 +33,7 @@ namespace pw::sync { /// The single consumer aspect of the API permits the use of a smaller and/or /// faster native APIs such as direct thread signaling. /// -/// The TimedThreadNotification is initialized to being empty (latch is not +/// The `TimedThreadNotification` is initialized to being empty (latch is not /// set). class TimedThreadNotification : public ThreadNotification { public: diff --git a/pw_sync/public/pw_sync/virtual_basic_lockable.h b/pw_sync/public/pw_sync/virtual_basic_lockable.h index 3a262b22d..501036535 100644 --- a/pw_sync/public/pw_sync/virtual_basic_lockable.h +++ b/pw_sync/public/pw_sync/virtual_basic_lockable.h @@ -18,10 +18,8 @@ namespace pw::sync { -/// @class VirtualBasicLockable -/// -/// The VirtualBasicLockable is a virtual lock abstraction for locks which meet -/// the C++ named BasicLockable requirements of lock() and unlock(). +/// The `VirtualBasicLockable` is a virtual lock abstraction for locks which +/// meet the C++ named BasicLockable requirements of lock() and unlock(). /// /// This virtual indirection is useful in case you need configurable lock /// selection in a portable module where the final type is not defined upstream @@ -46,14 +44,12 @@ class PW_LOCKABLE("pw::sync::VirtualBasicLockable") VirtualBasicLockable { private: /// Uses a single virtual method with an enum to minimize the vtable cost per - /// implementation of VirtualBasicLockable. + /// implementation of `VirtualBasicLockable`. virtual void DoLockOperation(Operation operation) = 0; }; -/// @class NoOpLock -/// -/// The NoOpLock is a type of VirtualBasicLockable that does nothing, i.e. lock -/// operations are no-ops. +/// The `NoOpLock` is a type of `VirtualBasicLockable` that does nothing, i.e. +/// lock operations are no-ops. class PW_LOCKABLE("pw::sync::NoOpLock") NoOpLock final : public VirtualBasicLockable { public: diff --git a/pw_sync_baremetal/BUILD.bazel b/pw_sync_baremetal/BUILD.bazel index a137c695f..d401d99d7 100644 --- a/pw_sync_baremetal/BUILD.bazel +++ b/pw_sync_baremetal/BUILD.bazel @@ -85,9 +85,9 @@ pw_cc_library( "public_overrides", ], target_compatible_with = ["@platforms//os:none"], - visibility = ["//visibility:private"], + visibility = ["//pw_sync:__pkg__"], deps = [ "//pw_assert", - "//pw_sync:recursive_mutex", + "//pw_sync:recursive_mutex_facade", ], ) diff --git a/pw_sync_freertos/BUILD.bazel b/pw_sync_freertos/BUILD.bazel index 5609b5f66..1be8b9faa 100644 --- a/pw_sync_freertos/BUILD.bazel +++ b/pw_sync_freertos/BUILD.bazel @@ -37,11 +37,10 @@ pw_cc_library( "//pw_build/constraints/rtos:freertos", ], deps = [ - # TODO(b/234876414): This should depend on FreeRTOS but our third parties currently - # do not have Bazel support. "//pw_assert", "//pw_chrono:system_clock", "//pw_chrono_freertos:system_clock_headers", + "@freertos", ], ) @@ -77,12 +76,11 @@ pw_cc_library( "//pw_build/constraints/rtos:freertos", ], deps = [ - # TODO(b/234876414): This should depend on FreeRTOS but our third parties currently - # do not have Bazel support. "//pw_assert", - "//pw_sync:counting_semaphore_facade", "//pw_chrono:system_clock", "//pw_chrono_freertos:system_clock_headers", + "//pw_sync:counting_semaphore_facade", + "@freertos", ], ) @@ -91,9 +89,17 @@ pw_cc_library( srcs = [ "counting_semaphore.cc", ], - target_compatible_with = [ - "//pw_build/constraints/rtos:freertos", - ], + target_compatible_with = select({ + # Not compatible with this FreeRTOS config, because it does not enable + # FreeRTOS counting semaphores. We mark it explicitly incompatible to + # that this library is skipped when you + # `bazel build //pw_sync_freertos/...` for a platform using that + # config. + "//targets/stm32f429i_disc1_stm32cube:freertos_config_cv": ["@platforms//:incompatible"], + "//conditions:default": [ + "//pw_build/constraints/rtos:freertos", + ], + }), deps = [ ":counting_semaphore_headers", "//pw_assert", @@ -118,9 +124,9 @@ pw_cc_library( "//pw_build/constraints/rtos:freertos", ], deps = [ - # TODO(b/234876414): This should depend on FreeRTOS but our third parties currently - # do not have Bazel support. "//pw_assert", + "//pw_interrupt:context", + "@freertos", ], ) @@ -153,12 +159,11 @@ pw_cc_library( ], deps = [ "//pw_assert", - # TODO(b/234876414): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. "//pw_interrupt:context", "//pw_polyfill", "//pw_sync:interrupt_spin_lock", "//pw_sync:lock_annotations", + "@freertos", ], ) @@ -192,10 +197,9 @@ pw_cc_library( "//pw_build/constraints/rtos:freertos", ], deps = [ - # TODO(b/234876414): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. "//pw_chrono:system_clock", "//pw_sync:timed_thread_notification_facade", + "@freertos", ], ) @@ -230,10 +234,9 @@ pw_cc_library( "//pw_build/constraints/rtos:freertos", ], deps = [ - # TODO(b/234876414): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. "//pw_chrono:system_clock", "//pw_sync:timed_mutex_facade", + "@freertos", ], ) @@ -246,6 +249,7 @@ pw_cc_library( "//pw_build/constraints/rtos:freertos", ], deps = [ + ":mutex_headers", ":timed_mutex_headers", "//pw_assert", "//pw_chrono_freertos:system_clock_headers", @@ -269,8 +273,9 @@ pw_cc_library( target_compatible_with = [ "//pw_build/constraints/rtos:freertos", ], - # TODO(b/234876414): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. + deps = [ + "@freertos", + ], ) pw_cc_library( diff --git a/pw_system/BUILD.bazel b/pw_system/BUILD.bazel index a54b21bd8..d9fb2e97c 100644 --- a/pw_system/BUILD.bazel +++ b/pw_system/BUILD.bazel @@ -263,26 +263,18 @@ pw_cc_library( ], ) -# TODO(b/234877642): This is broken out into a separate pw_cc_library target as -# a workaround for pw_cc_binary not supporting `select` in its deps. -pw_cc_library( - name = "boot", - deps = select({ - "//pw_build/constraints/rtos:freertos": [], - "//conditions:default": ["//targets/host_device_simulator:boot"], - }), -) - pw_cc_binary( name = "system_example", srcs = ["example_user_app_init.cc"], deps = [ - ":boot", ":init", ":io", ":target_hooks", "//pw_stream", "//pw_stream:sys_io_stream", "//pw_unit_test:rpc_service", - ], + ] + select({ + "//pw_build/constraints/rtos:freertos": [], + "//conditions:default": ["//targets/host_device_simulator:boot"], + }), ) diff --git a/pw_system/py/pw_system/console.py b/pw_system/py/pw_system/console.py index 80fd2c3f3..49950884a 100644 --- a/pw_system/py/pw_system/console.py +++ b/pw_system/py/pw_system/console.py @@ -51,7 +51,7 @@ from typing import ( ) import socket -import serial # type: ignore +import serial import IPython # type: ignore from pw_cli import log as pw_cli_log diff --git a/pw_system/py/setup.cfg b/pw_system/py/setup.cfg index 9db8b71e6..fea637c72 100644 --- a/pw_system/py/setup.cfg +++ b/pw_system/py/setup.cfg @@ -23,6 +23,7 @@ packages = find: zip_safe = False install_requires = pyserial>=3.5,<4.0 + types-pyserial>=3.5,<4.0 [options.entry_points] console_scripts = pw-system-console = pw_system.console:main diff --git a/pw_system/socket_target_io.cc b/pw_system/socket_target_io.cc index 55f105408..441de02b4 100644 --- a/pw_system/socket_target_io.cc +++ b/pw_system/socket_target_io.cc @@ -29,11 +29,15 @@ constexpr uint16_t kPort = PW_SYSTEM_SOCKET_IO_PORT; stream::SocketStream& GetStream() { static bool running = false; static std::mutex socket_open_lock; + static stream::ServerSocket server_socket; static stream::SocketStream socket_stream; std::lock_guard guard(socket_open_lock); if (!running) { printf("Awaiting connection on port %d\n", static_cast<int>(kPort)); - PW_CHECK_OK(socket_stream.Serve(kPort)); + PW_CHECK_OK(server_socket.Listen(kPort)); + auto accept_result = server_socket.Accept(); + PW_CHECK_OK(accept_result.status()); + socket_stream = *std::move(accept_result); printf("Client connected\n"); running = true; } diff --git a/pw_thread/BUILD.bazel b/pw_thread/BUILD.bazel index c61f0950f..c54848a8f 100644 --- a/pw_thread/BUILD.bazel +++ b/pw_thread/BUILD.bazel @@ -141,6 +141,7 @@ pw_cc_facade( includes = ["public"], deps = [ ":id_facade", + ":thread_core", ], ) diff --git a/pw_thread_freertos/BUILD.bazel b/pw_thread_freertos/BUILD.bazel index 3cc1e9362..c401c2a85 100644 --- a/pw_thread_freertos/BUILD.bazel +++ b/pw_thread_freertos/BUILD.bazel @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations under # the License. +load("@bazel_skylib//rules:run_binary.bzl", "run_binary") load( "//pw_build:pigweed.bzl", "pw_cc_facade", @@ -35,6 +36,10 @@ pw_cc_library( "id_public_overrides", "public", ], + deps = [ + "//pw_interrupt:context", + "@freertos", + ], ) pw_cc_library( @@ -46,8 +51,6 @@ pw_cc_library( ":id_headers", "//pw_thread:id_facade", ], - # TODO(b/234876414): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. ) pw_cc_library( @@ -78,10 +81,9 @@ pw_cc_library( "//pw_assert", "//pw_chrono:system_clock", "//pw_chrono_freertos:system_clock_headers", + "//pw_thread:id", "//pw_thread:sleep_facade", ], - # TODO(b/234876414): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. ) # This target provides the FreeRTOS specific headers needs for thread creation. @@ -108,13 +110,8 @@ pw_cc_library( "//pw_assert", "//pw_string", "//pw_sync:binary_semaphore", - - # There's a circular dependency here, need to have the header part of - # the facade visibile to this library. - "//pw_thread:thread", + "//pw_thread:thread_facade", ], - # TODO(b/234876414): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. ) pw_cc_library( @@ -130,8 +127,6 @@ pw_cc_library( ":thread_headers", "//pw_assert", ], - # TODO(b/234876414): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. ) pw_cc_library( @@ -152,6 +147,8 @@ pw_cc_library( pw_cc_test( name = "dynamic_thread_backend_test", + # TODO(b/271465588): Get this test to build. + tags = ["manual"], deps = [ ":dynamic_test_threads", "//pw_thread:thread_facade_test", @@ -176,6 +173,8 @@ pw_cc_library( pw_cc_test( name = "static_thread_backend_test", + # TODO(b/271465588): Get this test to build. + tags = ["manual"], deps = [ ":static_test_threads", "//pw_thread:thread_facade_test", @@ -195,8 +194,9 @@ pw_cc_library( target_compatible_with = [ "//pw_build/constraints/rtos:freertos", ], - # TODO(b/234876414): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. + deps = [ + "@freertos", + ], ) pw_cc_library( @@ -213,10 +213,6 @@ pw_cc_library( "pw_thread_freertos_private/thread_iteration.h", "thread_iteration.cc", ], - hdrs = [ - "public/pw_thread_freertos/thread_iteration.h", - "public_overrides/pw_thread_backend/thread_iteration.h", - ], target_compatible_with = [ "//pw_build/constraints/rtos:freertos", ], @@ -226,10 +222,9 @@ pw_cc_library( "//pw_span", "//pw_status", "//pw_thread:thread_info", + "//pw_thread:thread_iteration_facade", "//pw_thread_freertos:util", ], - # TODO(b/234876414): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. ) pw_cc_test( @@ -238,9 +233,8 @@ pw_cc_test( "pw_thread_freertos_private/thread_iteration.h", "thread_iteration_test.cc", ], - target_compatible_with = [ - "//pw_build/constraints/rtos:freertos", - ], + # TODO(b/271465588): Get this test to build. + tags = ["manual"], deps = [ ":freertos_tasktcb", ":static_test_threads", @@ -255,8 +249,6 @@ pw_cc_test( "//pw_thread:thread_info", "//pw_thread:thread_iteration", ], - # TODO(b/234876414): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. ) pw_cc_library( @@ -267,15 +259,16 @@ pw_cc_library( hdrs = [ "public/pw_thread_freertos/util.h", ], + includes = ["public"], target_compatible_with = [ "//pw_build/constraints/rtos:freertos", ], deps = [ "//pw_function", + "//pw_log", "//pw_status", + "@freertos", ], - # TODO(b/234876414): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. ) pw_cc_library( @@ -286,6 +279,11 @@ pw_cc_library( hdrs = [ "public/pw_thread_freertos/snapshot.h", ], + # TODO(b/269204725): Put this in the toolchain configuration instead. I + # would like to say `copts = ["-Wno-c++20-designator"]`, but arm-gcc tells + # me that's an "unrecognized command line option"; I think it may be a + # clang-only flag. + copts = ["-Wno-pedantic"], target_compatible_with = [ "//pw_build/constraints/rtos:freertos", ], @@ -299,8 +297,6 @@ pw_cc_library( "//pw_thread:snapshot", "//pw_thread:thread_cc.pwpb", ], - # TODO(b/234876414): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. ) pw_cc_facade( @@ -312,13 +308,28 @@ pw_cc_facade( target_compatible_with = [ "//pw_build/constraints/rtos:freertos", ], - # TODO(b/234876414): This should depend on FreeRTOS but our third parties - # currently do not have Bazel support. ) pw_cc_library( name = "freertos_tasktcb", + hdrs = [ + ":generate_freertos_tsktcb", + ], + includes = ["thread_public_overrides"], deps = [ ":freertos_tasktcb_facade", ], ) + +run_binary( + name = "generate_freertos_tsktcb", + srcs = [ + "@freertos//:tasks.c", + ], + outs = [":thread_public_overrides/pw_thread_freertos_backend/freertos_tsktcb.h"], + args = [ + "--freertos-tasks-c=$(location @freertos//:tasks.c)", + "--output=$(location :thread_public_overrides/pw_thread_freertos_backend/freertos_tsktcb.h)", + ], + tool = "//pw_thread_freertos/py:generate_freertos_tsktcb", +) diff --git a/pw_thread_freertos/py/BUILD.bazel b/pw_thread_freertos/py/BUILD.bazel new file mode 100644 index 000000000..a6b3f453d --- /dev/null +++ b/pw_thread_freertos/py/BUILD.bazel @@ -0,0 +1,21 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# Python utilities for code generation. + +py_binary( + name = "generate_freertos_tsktcb", + srcs = ["pw_thread_freertos/generate_freertos_tsktcb.py"], + visibility = ["//pw_thread_freertos:__subpackages__"], +) diff --git a/pw_thread_freertos/py/pw_thread_freertos/generate_freertos_tsktcb.py b/pw_thread_freertos/py/pw_thread_freertos/generate_freertos_tsktcb.py index 69a81a9ee..c37a376c9 100644 --- a/pw_thread_freertos/py/pw_thread_freertos/generate_freertos_tsktcb.py +++ b/pw_thread_freertos/py/pw_thread_freertos/generate_freertos_tsktcb.py @@ -20,7 +20,7 @@ the specified output path. import argparse import re import sys -from typing import TextIO +from typing import Optional, TextIO from pathlib import Path _GENERATED_HEADER = """\ @@ -40,7 +40,18 @@ def _parse_args() -> argparse.Namespace: parser.add_argument( '--freertos-src-dir', type=Path, - help='Path to the FreeRTOS source directory.', + help=( + 'Path to the FreeRTOS source directory. Required unless' + ' --freertos-tasks-c is provided.' + ), + ) + parser.add_argument( + '--freertos-tasks-c', + type=Path, + help=( + 'Path to the tasks.c file in the FreeRTOS source directory. ' + 'Required unless --freertos-src-dir is provided.' + ), ) parser.add_argument( '--output', @@ -62,8 +73,15 @@ def _extract_struct(tasks_src: str): raise ValueError('Could not find tskTCB struct in tasks.c') -def _main(freertos_src_dir: Path, output: TextIO): - with open(freertos_src_dir / 'tasks.c', 'r') as tasks_c: +def _main( + freertos_src_dir: Optional[Path], + freertos_tasks_c: Optional[Path], + output: TextIO, +): + if freertos_tasks_c is None or not freertos_tasks_c.is_file(): + assert freertos_src_dir is not None + freertos_tasks_c = freertos_src_dir / 'tasks.c' + with open(freertos_tasks_c, 'r') as tasks_c: output.write(_GENERATED_HEADER) output.write(_extract_struct(tasks_c.read())) diff --git a/pw_thread_zephyr/BUILD.gn b/pw_thread_zephyr/BUILD.gn new file mode 100644 index 000000000..8cbd4532f --- /dev/null +++ b/pw_thread_zephyr/BUILD.gn @@ -0,0 +1,25 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import("//build_overrides/pigweed.gni") + +import("$dir_pw_docgen/docs.gni") +import("$dir_pw_unit_test/test.gni") + +pw_doc_group("docs") { + sources = [ "docs.rst" ] +} + +pw_test_group("tests") { +} diff --git a/pw_thread_zephyr/CMakeLists.txt b/pw_thread_zephyr/CMakeLists.txt new file mode 100644 index 000000000..41e357392 --- /dev/null +++ b/pw_thread_zephyr/CMakeLists.txt @@ -0,0 +1,30 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +pw_add_library(pw_thread_zephyr.sleep STATIC + HEADERS + public/pw_thread_zephyr/sleep_inline.h + sleep_public_overrides/pw_thread_backend/sleep_inline.h + PUBLIC_INCLUDES + public + sleep_public_overrides + PUBLIC_DEPS + pw_chrono.system_clock + pw_thread.sleep.facade + SOURCES + sleep.cc + PRIVATE_DEPS + pw_chrono_zephyr.system_clock + pw_assert.check +) diff --git a/pw_thread_zephyr/Kconfig b/pw_thread_zephyr/Kconfig new file mode 100644 index 000000000..65b9c3ab8 --- /dev/null +++ b/pw_thread_zephyr/Kconfig @@ -0,0 +1,18 @@ +# Copyright 2023 The Pigweed Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +config PIGWEED_THREAD_SLEEP + bool "Enabled the Zephyr pw_thread.sleep backend" + select PIGWEED_CHRONO_SYSTEM_CLOCK + select PIGWEED_ASSERT diff --git a/pw_thread_zephyr/docs.rst b/pw_thread_zephyr/docs.rst new file mode 100644 index 000000000..9608eb149 --- /dev/null +++ b/pw_thread_zephyr/docs.rst @@ -0,0 +1,8 @@ +.. _module-pw_thread_zephyr: + +---------------- +pw_thread_zephyr +---------------- +This is a set of backends for pw_thread based on the Zephyr RTOS. Currently, +only the pw_thread.sleep facade is implemented which is enabled via +``CONFIG_PIGWEED_THREAD_SLEEP=y``. diff --git a/pw_thread_zephyr/public/pw_thread_zephyr/sleep_inline.h b/pw_thread_zephyr/public/pw_thread_zephyr/sleep_inline.h new file mode 100644 index 000000000..f7f9abd80 --- /dev/null +++ b/pw_thread_zephyr/public/pw_thread_zephyr/sleep_inline.h @@ -0,0 +1,28 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#pragma once + +#include <zephyr/kernel.h> +#include <zephyr/sys/util.h> + +#include "pw_chrono/system_clock.h" +#include "pw_thread/sleep.h" + +namespace pw::this_thread { + +inline void sleep_for(chrono::SystemClock::duration sleep_duration) { + sleep_until(chrono::SystemClock::TimePointAfterAtLeast(sleep_duration)); +} + +} // namespace pw::this_thread diff --git a/pw_thread_zephyr/sleep.cc b/pw_thread_zephyr/sleep.cc new file mode 100644 index 000000000..7a3c78d19 --- /dev/null +++ b/pw_thread_zephyr/sleep.cc @@ -0,0 +1,51 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#include "pw_thread/sleep.h" + +#include <algorithm> +#include <limits> + +#include "pw_assert/check.h" +#include "pw_chrono/system_clock.h" +#include "pw_chrono_zephyr/system_clock_constants.h" + +using pw::chrono::SystemClock; + +namespace pw::this_thread { + +void sleep_until(SystemClock::time_point wakeup_time) { + SystemClock::time_point now = chrono::SystemClock::now(); + + // Check if the expiration deadline has already passed, yield. + if (wakeup_time <= now) { + k_yield(); + return; + } + + // The maximum amount of time we should sleep for in a single command. + constexpr chrono::SystemClock::duration kMaxTimeoutMinusOne = + pw::chrono::zephyr::kMaxTimeout - SystemClock::duration(1); + + while (now < wakeup_time) { + // Sleep either the full remaining duration or the maximum timout + k_sleep(Z_TIMEOUT_TICKS( + std::min((wakeup_time - now).count(), kMaxTimeoutMinusOne.count()))); + + // Check how much time has passed, the scheduler can wake us up early. + now = SystemClock::now(); + } +} + +} // namespace pw::this_thread diff --git a/pw_thread_zephyr/sleep_public_overrides/pw_thread_backend/sleep_inline.h b/pw_thread_zephyr/sleep_public_overrides/pw_thread_backend/sleep_inline.h new file mode 100644 index 000000000..3dcf01136 --- /dev/null +++ b/pw_thread_zephyr/sleep_public_overrides/pw_thread_backend/sleep_inline.h @@ -0,0 +1,16 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#pragma once + +#include "pw_thread_zephyr/sleep_inline.h" diff --git a/pw_tokenizer/BUILD.bazel b/pw_tokenizer/BUILD.bazel index b776067ac..26ce5dcbb 100644 --- a/pw_tokenizer/BUILD.bazel +++ b/pw_tokenizer/BUILD.bazel @@ -213,6 +213,15 @@ pw_cc_fuzz_test( ) pw_cc_test( + name = "encode_args_test", + srcs = ["encode_args_test.cc"], + deps = [ + ":pw_tokenizer", + "//pw_unit_test", + ], +) + +pw_cc_test( name = "hash_test", srcs = [ "hash_test.cc", diff --git a/pw_tokenizer/BUILD.gn b/pw_tokenizer/BUILD.gn index abc73bb0a..fc276f456 100644 --- a/pw_tokenizer/BUILD.gn +++ b/pw_tokenizer/BUILD.gn @@ -37,7 +37,11 @@ config("public_include_path") { } config("linker_script") { - inputs = [ "pw_tokenizer_linker_sections.ld" ] + inputs = [ + "pw_tokenizer_linker_sections.ld", + "pw_tokenizer_linker_rules.ld", + ] + lib_dirs = [ "." ] # Automatically add the tokenizer linker sections when cross-compiling or # building for Linux. macOS and Windows executables are not supported. @@ -56,7 +60,6 @@ config("linker_script") { rebase_path("add_tokenizer_sections_to_default_script.ld", root_build_dir), ] - lib_dirs = [ "." ] inputs += [ "add_tokenizer_sections_to_default_script.ld" ] } @@ -169,17 +172,25 @@ pw_test_group("tests") { ":argument_types_test", ":base64_test", ":decode_test", - ":detokenize_fuzzer", + ":detokenize_fuzzer_test", ":detokenize_test", + ":encode_args_test", ":hash_test", ":simple_tokenize_test", - ":token_database_fuzzer", + ":token_database_fuzzer_test", ":token_database_test", ":tokenize_test", ] group_deps = [ "$dir_pw_preprocessor:tests" ] } +group("fuzzers") { + deps = [ + ":detokenize_fuzzer", + ":token_database_fuzzer", + ] +} + pw_test("argument_types_test") { sources = [ "argument_types_test.cc", @@ -226,6 +237,11 @@ pw_test("detokenize_test") { enable_if = pw_build_EXECUTABLE_TARGET_TYPE != "arduino_executable" } +pw_test("encode_args_test") { + sources = [ "encode_args_test.cc" ] + deps = [ ":pw_tokenizer" ] +} + pw_test("hash_test") { sources = [ "hash_test.cc", @@ -264,6 +280,7 @@ pw_fuzzer("token_database_fuzzer") { "$dir_pw_preprocessor", dir_pw_span, ] + enable_test_if = false } pw_fuzzer("detokenize_fuzzer") { diff --git a/pw_tokenizer/CMakeLists.txt b/pw_tokenizer/CMakeLists.txt index b3d52bd88..cd4a419b8 100644 --- a/pw_tokenizer/CMakeLists.txt +++ b/pw_tokenizer/CMakeLists.txt @@ -54,12 +54,15 @@ pw_add_library(pw_tokenizer STATIC pw_varint ) -if("${CMAKE_SYSTEM_NAME}" STREQUAL "") +if(Zephyr_FOUND) + zephyr_linker_sources(SECTIONS "${CMAKE_CURRENT_SOURCE_DIR}/pw_tokenizer_linker_rules.ld") +elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "") target_link_options(pw_tokenizer PUBLIC "-T${CMAKE_CURRENT_SOURCE_DIR}/pw_tokenizer_linker_sections.ld" + "-L${CMAKE_CURRENT_SOURCE_DIR}" ) -elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" OR Zephyr_FOUND) +elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") target_link_options(pw_tokenizer PUBLIC "-T${CMAKE_CURRENT_SOURCE_DIR}/add_tokenizer_sections_to_default_script.ld" @@ -169,6 +172,16 @@ pw_add_test(pw_tokenizer.detokenize_test pw_tokenizer ) +pw_add_test(pw_tokenizer.encode_args_test + SOURCES + encode_args_test.cc + PRIVATE_DEPS + pw_tokenizer + GROUPS + modules + pw_tokenizer +) + pw_add_test(pw_tokenizer.hash_test SOURCES hash_test.cc diff --git a/pw_tokenizer/argument_types_test.cc b/pw_tokenizer/argument_types_test.cc index 696886e37..10c309529 100644 --- a/pw_tokenizer/argument_types_test.cc +++ b/pw_tokenizer/argument_types_test.cc @@ -18,6 +18,7 @@ #include "gtest/gtest.h" #include "pw_preprocessor/concat.h" +#include "pw_tokenizer/tokenize.h" #include "pw_tokenizer_private/argument_types_test.h" namespace pw::tokenizer { diff --git a/pw_tokenizer/argument_types_test_c.c b/pw_tokenizer/argument_types_test_c.c index 4308fcb55..aa2b52ffb 100644 --- a/pw_tokenizer/argument_types_test_c.c +++ b/pw_tokenizer/argument_types_test_c.c @@ -18,6 +18,7 @@ #include <assert.h> #include <stddef.h> +#include "pw_tokenizer/tokenize.h" #include "pw_tokenizer_private/argument_types_test.h" #ifdef __cplusplus diff --git a/pw_tokenizer/database.gni b/pw_tokenizer/database.gni index f5cc21bb1..a0cb024e2 100644 --- a/pw_tokenizer/database.gni +++ b/pw_tokenizer/database.gni @@ -94,13 +94,12 @@ template("pw_tokenizer_database") { pw_python_action(target_name) { script = "$dir_pw_tokenizer/py/pw_tokenizer/database.py" - # Restrict parallelism for updating this database file to one thread. This - # makes it safe to update it from multiple toolchains. - pool = "$dir_pw_tokenizer/pool:database($default_toolchain)" - inputs = _input_databases if (_create == "") { + # Restrict parallelism for updating this database file to one thread. This + # makes it safe to update it from multiple toolchains. + pool = "$dir_pw_tokenizer/pool:database($default_toolchain)" args = [ "add" ] if (defined(invoker.commit)) { args += [ diff --git a/pw_tokenizer/docs.rst b/pw_tokenizer/docs.rst index dc4d66df3..944f4cb32 100644 --- a/pw_tokenizer/docs.rst +++ b/pw_tokenizer/docs.rst @@ -154,6 +154,11 @@ Once enabled, the tokenizer headers can be included like any Zephyr headers: #include <pw_tokenizer/tokenize.h> +.. note:: + Zephyr handles the additional linker sections via + ``pw_tokenizer_linker_rules.ld`` which is added to the end of the linker file + via a call to ``zephyr_linker_sources(SECTIONS ...)``. + ------------ Tokenization ------------ @@ -161,6 +166,8 @@ Tokenization converts a string literal to a token. If it's a printf-style string, its arguments are encoded along with it. The results of tokenization can be sent off device or stored in place of a full string. +.. doxygentypedef:: pw_tokenizer_Token + Tokenization macros =================== Adding tokenization to a project is simple. To tokenize a string, include @@ -171,25 +178,9 @@ Tokenize a string literal ``pw_tokenizer`` provides macros for tokenizing string literals with no arguments. -.. c:macro:: PW_TOKENIZE_STRING(string_literal) - - Converts a string literal to a ``uint32_t`` token in a standalone statement. - C and C++ compatible. - - .. code-block:: cpp - - constexpr uint32_t token = PW_TOKENIZE_STRING("Any string literal!"); - -.. c:macro:: PW_TOKENIZE_STRING_DOMAIN(domain, string_literal) - - Tokenizes a string literal in a standalone statement using the specified - :ref:`domain <module-pw_tokenizer-domains>`. C and C++ compatible. - -.. c:macro:: PW_TOKENIZE_STRING_MASK(domain, mask, string_literal) - - Tokenizes a string literal in a standalone stateemnt using the specified - :ref:`domain <module-pw_tokenizer-domains>` and :ref:`bit mask - <module-pw_tokenizer-masks>`. C and C++ compatible. +.. doxygendefine:: PW_TOKENIZE_STRING +.. doxygendefine:: PW_TOKENIZE_STRING_DOMAIN +.. doxygendefine:: PW_TOKENIZE_STRING_MASK The tokenization macros above cannot be used inside other expressions. @@ -220,25 +211,9 @@ use of lambda functions, so while they can be used inside expressions, they require C++ and cannot be assigned to constexpr variables or be used with special function variables like ``__func__``. -.. c:macro:: PW_TOKENIZE_STRING_EXPR(string_literal) - - Converts a string literal to a ``uint32_t`` token within an expression. - Requires C++. - - .. code-block:: cpp - - DoSomething(PW_TOKENIZE_STRING_EXPR("Succeed")); - -.. c:macro:: PW_TOKENIZE_STRING_DOMAIN_EXPR(domain, string_literal) - - Tokenizes a string literal using the specified :ref:`domain - <module-pw_tokenizer-domains>` within an expression. Requires C++. - -.. c:macro:: PW_TOKENIZE_STRING_MASK_EXPR(domain, mask, string_literal) - - Tokenizes a string literal using the specified :ref:`domain - <module-pw_tokenizer-domains>` and :ref:`bit mask - <module-pw_tokenizer-masks>` within an expression. Requires C++. +.. doxygendefine:: PW_TOKENIZE_STRING_EXPR +.. doxygendefine:: PW_TOKENIZE_STRING_DOMAIN_EXPR +.. doxygendefine:: PW_TOKENIZE_STRING_MASK_EXPR .. admonition:: When to use these macros @@ -288,30 +263,16 @@ handle tokenized data in a function of their choosing. ``pw_tokenizer`` provides two low-level macros for projects to use to create custom tokenization macros. -.. c:macro:: PW_TOKENIZE_FORMAT_STRING(domain, mask, format_string, ...) - - Tokenizes a format string and sets the ``_pw_tokenizer_token`` variable to the - token. Must be used in its own scope, since the same variable is used in every - invocation. - - The tokenized string uses the specified :ref:`tokenization domain - <module-pw_tokenizer-domains>`. Use ``PW_TOKENIZER_DEFAULT_DOMAIN`` for the - default. The token also may be masked; use ``UINT32_MAX`` to keep all bits. - -.. c:macro:: PW_TOKENIZER_ARG_TYPES(...) - - Converts a series of arguments to a compact format that replaces the format - string literal. Evaluates to a ``pw_tokenizer_ArgTypes`` value. +.. doxygendefine:: PW_TOKENIZE_FORMAT_STRING +.. doxygendefine:: PW_TOKENIZER_ARG_TYPES The outputs of these macros are typically passed to an encoding function. That function encodes the token, argument types, and argument data to a buffer using helpers provided by ``pw_tokenizer/encode_args.h``. .. doxygenfunction:: pw::tokenizer::EncodeArgs - .. doxygenclass:: pw::tokenizer::EncodedMessage :members: - .. doxygenfunction:: pw_tokenizer_EncodeArgs Example @@ -382,18 +343,9 @@ stored as needed. Tokenize a message with arguments to a buffer --------------------------------------------- -.. c:macro:: PW_TOKENIZE_TO_BUFFER(buffer_pointer, buffer_size_pointer, format_string, arguments...) - - ``PW_TOKENIZE_TO_BUFFER`` encodes to a caller-provided buffer. - - .. code-block:: cpp - - uint8_t buffer[BUFFER_SIZE]; - size_t size_bytes = sizeof(buffer); - PW_TOKENIZE_TO_BUFFER(buffer, &size_bytes, format_string_literal, arguments...); - - While ``PW_TOKENIZE_TO_BUFFER`` is very flexible, it must be passed a buffer, - which increases its call site overhead. +.. doxygendefine:: PW_TOKENIZE_TO_BUFFER +.. doxygendefine:: PW_TOKENIZE_TO_BUFFER_DOMAIN +.. doxygendefine:: PW_TOKENIZE_TO_BUFFER_MASK .. admonition:: Why use this macro @@ -477,6 +429,10 @@ Arguments are encoded as follows: arguments short or avoid encoding them as strings (e.g. encode an enum as an integer instead of a string). See also `Tokenized strings as %s arguments`_. +Buffer sizing helper +-------------------- +.. doxygenfunction:: pw::tokenizer::MinEncodingBufferSizeBytes + Encoding command line utility ----------------------------- The ``pw_tokenizer.encode`` command line tool can be used to encode tokenized diff --git a/pw_tokenizer/encode_args_test.cc b/pw_tokenizer/encode_args_test.cc new file mode 100644 index 000000000..131f27b9a --- /dev/null +++ b/pw_tokenizer/encode_args_test.cc @@ -0,0 +1,42 @@ +// Copyright 2022 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#include "pw_tokenizer/encode_args.h" + +#include "gtest/gtest.h" + +namespace pw { +namespace tokenizer { + +static_assert(MinEncodingBufferSizeBytes<>() == 4); +static_assert(MinEncodingBufferSizeBytes<bool>() == 4 + 2); +static_assert(MinEncodingBufferSizeBytes<char>() == 4 + 2); +static_assert(MinEncodingBufferSizeBytes<short>() == 4 + 3); +static_assert(MinEncodingBufferSizeBytes<int>() == 4 + 5); +static_assert(MinEncodingBufferSizeBytes<long long>() == 4 + 10); +static_assert(MinEncodingBufferSizeBytes<float>() == 4 + 4); +static_assert(MinEncodingBufferSizeBytes<double>() == 4 + 4); +static_assert(MinEncodingBufferSizeBytes<const char*>() == 4 + 1); +static_assert(MinEncodingBufferSizeBytes<void*>() == 4 + 5 || + MinEncodingBufferSizeBytes<void*>() == 4 + 10); + +static_assert(MinEncodingBufferSizeBytes<int, double>() == 4 + 5 + 4); +static_assert(MinEncodingBufferSizeBytes<int, int, const char*>() == + 4 + 5 + 5 + 1); +static_assert( + MinEncodingBufferSizeBytes<const char*, long long, int, short>() == + 4 + 1 + 10 + 5 + 3); + +} // namespace tokenizer +} // namespace pw diff --git a/pw_tokenizer/public/pw_tokenizer/encode_args.h b/pw_tokenizer/public/pw_tokenizer/encode_args.h index 43124e51f..07e6e3192 100644 --- a/pw_tokenizer/public/pw_tokenizer/encode_args.h +++ b/pw_tokenizer/public/pw_tokenizer/encode_args.h @@ -25,12 +25,52 @@ #include <cstring> +#include "pw_polyfill/standard.h" #include "pw_span/span.h" #include "pw_tokenizer/config.h" #include "pw_tokenizer/tokenize.h" -namespace pw { -namespace tokenizer { +namespace pw::tokenizer { +namespace internal { + +// Returns the maximum encoded size of an argument of the specified type. +template <typename T> +constexpr size_t ArgEncodedSizeBytes() { + constexpr pw_tokenizer_ArgTypes kType = VarargsType<T>(); + if constexpr (kType == PW_TOKENIZER_ARG_TYPE_DOUBLE) { + return sizeof(float); + } else if constexpr (kType == PW_TOKENIZER_ARG_TYPE_STRING) { + return 1; // Size of the length byte only + } else if constexpr (kType == PW_TOKENIZER_ARG_TYPE_INT64) { + return 10; // Max size of a varint-encoded 64-bit integer + } else if constexpr (kType == PW_TOKENIZER_ARG_TYPE_INT) { + return sizeof(T) + 1; // Max size of zig-zag varint integer <= 32-bits + } else { + static_assert(sizeof(T) != sizeof(T), "Unsupported argument type"); + } +} + +} // namespace internal + +/// Calculates the minimum buffer size to allocate that is guaranteed to support +/// encoding the specified arguments. +/// +/// The contents of strings are NOT included in this total. The string's +/// length/status byte is guaranteed to fit, but the string contents may be +/// truncated. Encoding is considered to succeed as long as the string's +/// length/status byte is written, even if the actual string is truncated. +/// +/// Examples: +/// +/// - Message with no arguments: +/// `MinEncodingBufferSizeBytes() == 4` +/// - Message with an int argument +/// `MinEncodingBufferSizeBytes<int>() == 9 (4 + 5)` +template <typename... ArgTypes> +constexpr size_t MinEncodingBufferSizeBytes() { + return (sizeof(pw_tokenizer_Token) + ... + + internal::ArgEncodedSizeBytes<ArgTypes>()); +} /// Encodes a tokenized string's arguments to a buffer. The /// @cpp_type{pw_tokenizer_ArgTypes} parameter specifies the argument types, in @@ -97,8 +137,7 @@ class EncodedMessage { size_t size_; }; -} // namespace tokenizer -} // namespace pw +} // namespace pw::tokenizer #endif // PW_CXX_STANDARD_IS_SUPPORTED(17) diff --git a/pw_tokenizer/public/pw_tokenizer/internal/argument_types.h b/pw_tokenizer/public/pw_tokenizer/internal/argument_types.h index 0fdf96bc5..64c9a4cb1 100644 --- a/pw_tokenizer/public/pw_tokenizer/internal/argument_types.h +++ b/pw_tokenizer/public/pw_tokenizer/internal/argument_types.h @@ -174,14 +174,4 @@ constexpr pw_tokenizer_ArgTypes VarargsType() { #endif // __cplusplus -// Encodes the types of the provided arguments as a pw_tokenizer_ArgTypes -// value. Depending on the size of pw_tokenizer_ArgTypes, the bottom 4 or 6 -// bits store the number of arguments and the remaining bits store the types, -// two bits per type. -// -// The arguments are not evaluated; only their types are used to -// select the set their corresponding PW_TOKENIZER_ARG_TYPEs. -#define PW_TOKENIZER_ARG_TYPES(...) \ - PW_DELEGATE_BY_ARG_COUNT(_PW_TOKENIZER_TYPES_, __VA_ARGS__) - #define _PW_TOKENIZER_TYPES_0() ((pw_tokenizer_ArgTypes)0) diff --git a/pw_tokenizer/public/pw_tokenizer/tokenize.h b/pw_tokenizer/public/pw_tokenizer/tokenize.h index fd9671516..6b8e62c57 100644 --- a/pw_tokenizer/public/pw_tokenizer/tokenize.h +++ b/pw_tokenizer/public/pw_tokenizer/tokenize.h @@ -33,58 +33,54 @@ #include "pw_tokenizer/internal/argument_types.h" #include "pw_tokenizer/internal/tokenize_string.h" -// The type of the token used in place of a format string. Also available as -// pw::tokenizer::Token. +/// The type of the 32-bit token used in place of a string. Also available as +/// `pw::tokenizer::Token`. typedef uint32_t pw_tokenizer_Token; -// Strings may optionally be tokenized to a domain. Strings in different domains -// can be processed separately by the token database tools. Each domain in use -// must have a corresponding section declared in the linker script. See -// pw_tokenizer_linker_sections.ld for more details. +// Strings may optionally be tokenized to a domain. Strings in different +// domains can be processed separately by the token database tools. Each domain +// in use must have a corresponding section declared in the linker script. See +// `pw_tokenizer_linker_sections.ld` for more details. // // The default domain is an empty string. #define PW_TOKENIZER_DEFAULT_DOMAIN "" -// Tokenizes a string and converts it to a pw_tokenizer_Token. In C++, the -// string may be a literal or a constexpr char array. In C, the argument must be -// a string literal. In either case, the string must be null terminated, but may -// contain any characters (including '\0'). -// -// Two different versions are provided, PW_TOKENIZE_STRING and -// PW_TOKENIZE_STRING_EXPR. PW_TOKENIZE_STRING can be assigned to a local or -// global variable, including constexpr variables. PW_TOKENIZE_STRING can be -// used with special function variables like __func__. -// -// PW_TOKENIZE_STRING_EXPR can be used inside an expression. -// PW_TOKENIZE_STRING_EXPR is implemented using a lambda function, so it will -// not work as expected with special function variables like __func__. It is -// also only usable with C++. -// -// constexpr uint32_t global = PW_TOKENIZE_STRING("Wow!"); // This works. -// -// void SomeFunction() { -// constexpr uint32_t token = PW_TOKENIZE_STRING("Cool!"); // This works. -// -// DoSomethingElse(PW_TOKENIZE_STRING("Lame!")); // This does NOT work. -// DoSomethingElse(PW_TOKENIZE_STRING_EXPR("Yay!")); // This works. -// -// constexpr uint32_t token2 = PW_TOKENIZE_STRING(__func__); // This works. -// DoSomethingElse(PW_TOKENIZE_STRING_EXPR(__func__)); // Does NOT work. -// } -// +/// Converts a string literal to a `pw_tokenizer_Token` (`uint32_t`) token in a +/// standalone statement. C and C++ compatible. In C++, the string may be a +/// literal or a constexpr char array, including function variables like +/// `__func__`. In C, the argument must be a string literal. In either case, the +/// string must be null terminated, but may contain any characters (including +/// '\0'). +/// +/// @code +/// +/// constexpr uint32_t token = PW_TOKENIZE_STRING("Any string literal!"); +/// +/// @endcode #define PW_TOKENIZE_STRING(string_literal) \ PW_TOKENIZE_STRING_DOMAIN(PW_TOKENIZER_DEFAULT_DOMAIN, string_literal) +/// Converts a string literal to a ``uint32_t`` token within an expression. +/// Requires C++. +/// +/// @code +/// +/// DoSomething(PW_TOKENIZE_STRING_EXPR("Succeed")); +/// +/// @endcode #define PW_TOKENIZE_STRING_EXPR(string_literal) \ [&] { \ constexpr uint32_t lambda_ret_token = PW_TOKENIZE_STRING(string_literal); \ return lambda_ret_token; \ }() -// Same as PW_TOKENIZE_STRING, but tokenizes to the specified domain. +/// Tokenizes a string literal in a standalone statement using the specified +/// @rstref{domain <module-pw_tokenizer-domains>}. C and C++ compatible. #define PW_TOKENIZE_STRING_DOMAIN(domain, string_literal) \ PW_TOKENIZE_STRING_MASK(domain, UINT32_MAX, string_literal) +/// Tokenizes a string literal using the specified @rstref{domain +/// <module-pw_tokenizer-domains>} within an expression. Requires C++. #define PW_TOKENIZE_STRING_DOMAIN_EXPR(domain, string_literal) \ [&] { \ constexpr uint32_t lambda_ret_token = \ @@ -92,7 +88,9 @@ typedef uint32_t pw_tokenizer_Token; return lambda_ret_token; \ }() -// Same as PW_TOKENIZE_STRING_DOMAIN, but applies a mask to the token. +/// Tokenizes a string literal in a standalone statement using the specified +/// @rstref{domain <module-pw_tokenizer-domains>} and @rstref{bit mask +/// <module-pw_tokenizer-masks>}. C and C++ compatible. #define PW_TOKENIZE_STRING_MASK(domain, mask, string_literal) \ /* assign to a variable */ _PW_TOKENIZER_MASK_TOKEN(mask, string_literal); \ \ @@ -102,6 +100,9 @@ typedef uint32_t pw_tokenizer_Token; _PW_TOKENIZER_RECORD_ORIGINAL_STRING( \ _PW_TOKENIZER_MASK_TOKEN(mask, string_literal), domain, string_literal) +/// Tokenizes a string literal using the specified @rstref{domain +/// <module-pw_tokenizer-domains>} and @rstref{bit mask +/// <module-pw_tokenizer-masks>} within an expression. Requires C++. #define PW_TOKENIZE_STRING_MASK_EXPR(domain, mask, string_literal) \ [&] { \ constexpr uint32_t lambda_ret_token = \ @@ -112,26 +113,36 @@ typedef uint32_t pw_tokenizer_Token; #define _PW_TOKENIZER_MASK_TOKEN(mask, string_literal) \ ((pw_tokenizer_Token)(mask)&PW_TOKENIZER_STRING_TOKEN(string_literal)) -// Encodes a tokenized string and arguments to the provided buffer. The size of -// the buffer is passed via a pointer to a size_t. After encoding is complete, -// the size_t is set to the number of bytes written to the buffer. -// -// The macro's arguments are equivalent to the following function signature: -// -// TokenizeToBuffer(void* buffer, -// size_t* buffer_size_pointer, -// const char* format, -// ...); /* printf-style arguments */ -// -// For example, the following encodes a tokenized string with a temperature to a -// buffer. The buffer is passed to a function to send the message over a UART. -// -// uint8_t buffer[32]; -// size_t size_bytes = sizeof(buffer); -// PW_TOKENIZE_TO_BUFFER( -// buffer, &size_bytes, "Temperature (C): %0.2f", temperature_c); -// MyProject_EnqueueMessageForUart(buffer, size); -// +/// Encodes a tokenized string and arguments to the provided buffer. The size of +/// the buffer is passed via a pointer to a `size_t`. After encoding is +/// complete, the `size_t` is set to the number of bytes written to the buffer. +/// +/// The macro's arguments are equivalent to the following function signature: +/// +/// @code +/// +/// TokenizeToBuffer(void* buffer, +/// size_t* buffer_size_pointer, +/// const char* format, +/// ...); // printf-style arguments +/// @endcode +/// +/// For example, the following encodes a tokenized string with a temperature to +/// a buffer. The buffer is passed to a function to send the message over a +/// UART. +/// +/// @code +/// +/// uint8_t buffer[32]; +/// size_t size_bytes = sizeof(buffer); +/// PW_TOKENIZE_TO_BUFFER( +/// buffer, &size_bytes, "Temperature (C): %0.2f", temperature_c); +/// MyProject_EnqueueMessageForUart(buffer, size); +/// +/// @endcode +/// +/// While `PW_TOKENIZE_TO_BUFFER` is very flexible, it must be passed a buffer, +/// which increases its code size footprint at the call site. #define PW_TOKENIZE_TO_BUFFER(buffer, buffer_size_pointer, format, ...) \ PW_TOKENIZE_TO_BUFFER_DOMAIN(PW_TOKENIZER_DEFAULT_DOMAIN, \ buffer, \ @@ -139,13 +150,15 @@ typedef uint32_t pw_tokenizer_Token; format, \ __VA_ARGS__) -// Same as PW_TOKENIZE_TO_BUFFER, but tokenizes to the specified domain. +/// Same as @c_macro{PW_TOKENIZE_TO_BUFFER}, but tokenizes to the specified +/// @rstref{domain <module-pw_tokenizer-domains>}. #define PW_TOKENIZE_TO_BUFFER_DOMAIN( \ domain, buffer, buffer_size_pointer, format, ...) \ PW_TOKENIZE_TO_BUFFER_MASK( \ domain, UINT32_MAX, buffer, buffer_size_pointer, format, __VA_ARGS__) -// Same as PW_TOKENIZE_TO_BUFFER_DOMAIN, but applies a mask to the token. +/// Same as @c_macro{PW_TOKENIZE_TO_BUFFER_DOMAIN}, but applies a +/// @rstref{bit mask <module-pw_tokenizer-masks>} to the token. #define PW_TOKENIZE_TO_BUFFER_MASK( \ domain, mask, buffer, buffer_size_pointer, format, ...) \ do { \ @@ -157,6 +170,15 @@ typedef uint32_t pw_tokenizer_Token; PW_COMMA_ARGS(__VA_ARGS__)); \ } while (0) +/// Converts a series of arguments to a compact format that replaces the format +/// string literal. Evaluates to a `pw_tokenizer_ArgTypes` value. +/// +/// Depending on the size of `pw_tokenizer_ArgTypes`, the bottom 4 or 6 bits +/// store the number of arguments and the remaining bits store the types, two +/// bits per type. The arguments are not evaluated; only their types are used. +#define PW_TOKENIZER_ARG_TYPES(...) \ + PW_DELEGATE_BY_ARG_COUNT(_PW_TOKENIZER_TYPES_, __VA_ARGS__) + PW_EXTERN_C_START // These functions encode the tokenized strings. These should not be called @@ -177,12 +199,17 @@ static inline void pw_tokenizer_CheckFormatString(const char* format, ...) { PW_EXTERN_C_END -// These macros implement string tokenization. They should not be used directly; -// use one of the PW_TOKENIZE_* macros above instead. - -// This macro takes a printf-style format string and corresponding arguments. It -// checks that the arguments are correct, stores the format string in a special -// section, and calculates the string's token at compile time. This +/// Tokenizes a format string with optional arguments and sets the +/// `_pw_tokenizer_token` variable to the token. Must be used in its own scope, +/// since the same variable is used in every invocation. +/// +/// The tokenized string uses the specified @rstref{tokenization domain +/// <module-pw_tokenizer-domains>}. Use `PW_TOKENIZER_DEFAULT_DOMAIN` for the +/// default. The token also may be masked; use `UINT32_MAX` to keep all bits. +/// +/// This macro checks that the printf-style format string matches the arguments, +/// stores the format string in a special section, and calculates the string's +/// token at compile time. // clang-format off #define PW_TOKENIZE_FORMAT_STRING(domain, mask, format, ...) \ if (0) { /* Do not execute to prevent double evaluation of the arguments. */ \ diff --git a/pw_tokenizer/pw_tokenizer_linker_rules.ld b/pw_tokenizer/pw_tokenizer_linker_rules.ld new file mode 100644 index 000000000..d1f0aa796 --- /dev/null +++ b/pw_tokenizer/pw_tokenizer_linker_rules.ld @@ -0,0 +1,55 @@ +/* + * Copyright 2020 The Pigweed Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/* + * This file is separate from pw_tokenizer_linker_sections.ld because Zephyr + * already defines the top level SECTIONS label and requires new linker + * scripts to only add the individual sections. + */ + +/* + * This section stores metadata that may be used during tokenized string + * decoding. This metadata describes properties that may affect how the + * tokenized string is encoded or decoded -- the maximum length of the hash + * function and the sizes of certain integer types. + * + * Metadata is declared as key-value pairs. See the metadata variable in + * tokenize.cc for further details. + */ +.pw_tokenizer.info 0x0 (INFO) : +{ + KEEP(*(.pw_tokenizer.info)) +} + +/* + * Tokenized string entries are stored in this section. Each entry contains + * the original string literal and the calculated token that represents it. In + * the compiled code, the token and a compact argument list encoded in a + * uint32_t are used in place of the format string. The compiled code + * contains no references to the tokenized string entries in this section. + * + * The tokenized string entry format is specified by the + * pw::tokenizer::internal::Entry class in + * pw_tokenizer/public/pw_tokenizer/internal/tokenize_string.h. + * + * The section contents are declared with KEEP so that they are not removed + * from the ELF. These are never emitted in the final binary or loaded into + * memory. + */ +.pw_tokenizer.entries 0x0 (INFO) : +{ + KEEP(*(.pw_tokenizer.entries.*)) +} diff --git a/pw_tokenizer/pw_tokenizer_linker_sections.ld b/pw_tokenizer/pw_tokenizer_linker_sections.ld index ae17f47cd..a48a804cc 100644 --- a/pw_tokenizer/pw_tokenizer_linker_sections.ld +++ b/pw_tokenizer/pw_tokenizer_linker_sections.ld @@ -34,37 +34,5 @@ SECTIONS { - /* - * This section stores metadata that may be used during tokenized string - * decoding. This metadata describes properties that may affect how the - * tokenized string is encoded or decoded -- the maximum length of the hash - * function and the sizes of certain integer types. - * - * Metadata is declared as key-value pairs. See the metadata variable in - * tokenize.cc for further details. - */ - .pw_tokenizer.info 0x0 (INFO) : - { - KEEP(*(.pw_tokenizer.info)) - } - - /* - * Tokenized string entries are stored in this section. Each entry contains - * the original string literal and the calculated token that represents it. In - * the compiled code, the token and a compact argument list encoded in a - * uint32_t are used in place of the format string. The compiled code - * contains no references to the tokenized string entries in this section. - * - * The tokenized string entry format is specified by the - * pw::tokenizer::internal::Entry class in - * pw_tokenizer/public/pw_tokenizer/internal/tokenize_string.h. - * - * The section contents are declared with KEEP so that they are not removed - * from the ELF. These are never emitted in the final binary or loaded into - * memory. - */ - .pw_tokenizer.entries 0x0 (INFO) : - { - KEEP(*(.pw_tokenizer.entries.*)) - } + INCLUDE pw_tokenizer_linker_rules.ld } diff --git a/pw_tokenizer/py/pw_tokenizer/database.py b/pw_tokenizer/py/pw_tokenizer/database.py index 093186873..26a32a7fa 100755 --- a/pw_tokenizer/py/pw_tokenizer/database.py +++ b/pw_tokenizer/py/pw_tokenizer/database.py @@ -594,7 +594,7 @@ def _parse_args(): def replacement(value: str) -> Tuple[Pattern, 'str']: try: find, sub = unescaped_slash.split(value, 1) - except ValueError as err: + except ValueError as _err: raise argparse.ArgumentTypeError( 'replacements must be specified as "search_regex/replacement"' ) diff --git a/pw_tokenizer/py/pw_tokenizer/decode.py b/pw_tokenizer/py/pw_tokenizer/decode.py index 426350706..4796fb05e 100644 --- a/pw_tokenizer/py/pw_tokenizer/decode.py +++ b/pw_tokenizer/py/pw_tokenizer/decode.py @@ -787,7 +787,7 @@ class FormatString: # Start with the part of the format string up to the first specifier. string_pieces = [self.format_string[: spec_spans[0][0]]] - for ((_, end1), (start2, _)) in zip(spec_spans[:-1], spec_spans[1:]): + for (_, end1), (start2, _) in zip(spec_spans[:-1], spec_spans[1:]): string_pieces.append(self.format_string[end1:start2]) # Append the format string segment after the last format specifier. diff --git a/pw_tokenizer/py/pw_tokenizer/detokenize.py b/pw_tokenizer/py/pw_tokenizer/detokenize.py index fa6021241..3aa7a3a8b 100755 --- a/pw_tokenizer/py/pw_tokenizer/detokenize.py +++ b/pw_tokenizer/py/pw_tokenizer/detokenize.py @@ -73,6 +73,8 @@ ENCODED_TOKEN = struct.Struct('<I') BASE64_PREFIX = encode.BASE64_PREFIX.encode() DEFAULT_RECURSION = 9 +_RawIO = Union[io.RawIOBase, BinaryIO] + class DetokenizedString: """A detokenized string, with all results if there are collisions.""" @@ -264,7 +266,7 @@ class Detokenizer: def detokenize_base64_live( self, - input_file: BinaryIO, + input_file: _RawIO, output: BinaryIO, prefix: Union[str, bytes] = BASE64_PREFIX, recursion: int = DEFAULT_RECURSION, @@ -320,6 +322,7 @@ class Detokenizer: _PathOrStr = Union[Path, str] + # TODO(b/265334753): Reuse this function in database.py:LoadTokenDatabases def _parse_domain(path: _PathOrStr) -> Tuple[Path, Optional[Pattern[str]]]: """Extracts an optional domain regex pattern suffix from a path""" @@ -427,16 +430,14 @@ class PrefixedMessageDecoder: self.data = bytearray() - def _read_next(self, fd: BinaryIO) -> Tuple[bytes, int]: + def _read_next(self, fd: _RawIO) -> Tuple[bytes, int]: """Returns the next character and its index.""" - char = fd.read(1) + char = fd.read(1) or b'' index = len(self.data) self.data += char return char, index - def read_messages( - self, binary_fd: BinaryIO - ) -> Iterator[Tuple[bool, bytes]]: + def read_messages(self, binary_fd: _RawIO) -> Iterator[Tuple[bool, bytes]]: """Parses prefixed messages; yields (is_message, contents) chunks.""" message_start = None @@ -463,7 +464,7 @@ class PrefixedMessageDecoder: yield False, char def transform( - self, binary_fd: BinaryIO, transform: Callable[[bytes], bytes] + self, binary_fd: _RawIO, transform: Callable[[bytes], bytes] ) -> Iterator[bytes]: """Yields the file with a transformation applied to the messages.""" for is_message, chunk in self.read_messages(binary_fd): diff --git a/pw_tokenizer/py/pw_tokenizer/serial_detokenizer.py b/pw_tokenizer/py/pw_tokenizer/serial_detokenizer.py index 793f95b26..ab673a55e 100644 --- a/pw_tokenizer/py/pw_tokenizer/serial_detokenizer.py +++ b/pw_tokenizer/py/pw_tokenizer/serial_detokenizer.py @@ -21,7 +21,7 @@ import argparse import sys from typing import BinaryIO, Iterable -import serial # type: ignore +import serial from pw_tokenizer import database, detokenize, tokens @@ -80,7 +80,7 @@ def _parse_args(): def _detokenize_serial( databases: Iterable, - device: serial.Serial, + device: str, baudrate: int, show_errors: bool, output: BinaryIO, diff --git a/pw_tokenizer/py/setup.cfg b/pw_tokenizer/py/setup.cfg index 99e075d61..f7a20b7a7 100644 --- a/pw_tokenizer/py/setup.cfg +++ b/pw_tokenizer/py/setup.cfg @@ -21,6 +21,9 @@ description = Tools for working with tokenized strings [options] packages = pw_tokenizer zip_safe = False +install_requires = + pyserial>=3.5,<4.0 + types-pyserial>=3.5,<4.0 [options.package_data] pw_tokenizer = py.typed diff --git a/pw_tokenizer/ts/detokenizer.ts b/pw_tokenizer/ts/detokenizer.ts index 90bff2b1e..fe6ea910a 100644 --- a/pw_tokenizer/ts/detokenizer.ts +++ b/pw_tokenizer/ts/detokenizer.ts @@ -115,7 +115,7 @@ export class Detokenizer { data.byteOffset, 4 ).getUint32(0, true); - const args = new Uint8Array(data.buffer.slice(4)); + const args = new Uint8Array(data.buffer.slice(data.byteOffset + 4)); return {token, args}; } diff --git a/pw_tokenizer/ts/int_testdata.ts b/pw_tokenizer/ts/int_testdata.ts new file mode 100644 index 000000000..36b5a21be --- /dev/null +++ b/pw_tokenizer/ts/int_testdata.ts @@ -0,0 +1,52 @@ +// Copyright 2023 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +const IntDB = [ + ["%d", "-128", "%u", "4294967168", '\xff\x01'], + ["%d", "-10", "%u", "4294967286", '\x13'], + ["%d", "-9", "%u", "4294967287", '\x11'], + ["%d", "-8", "%u", "4294967288", '\x0f'], + ["%d", "-7", "%u", "4294967289", '\x0d'], + ["%d", "-6", "%u", "4294967290", '\x0b'], + ["%d", "-5", "%u", "4294967291", '\x09'], + ["%d", "-4", "%u", "4294967292", '\x07'], + ["%d", "-3", "%u", "4294967293", '\x05'], + ["%d", "-2", "%u", "4294967294", '\x03'], + ["%d", "-1", "%u", "4294967295", '\x01'], + ["%d", "0", "%u", "0", '\x00'], + ["%d", "1", "%u", "1", '\x02'], + ["%d", "2", "%u", "2", '\x04'], + ["%d", "3", "%u", "3", '\x06'], + ["%d", "4", "%u", "4", '\x08'], + ["%d", "5", "%u", "5", '\x0a'], + ["%d", "6", "%u", "6", '\x0c'], + ["%d", "7", "%u", "7", '\x0e'], + ["%d", "8", "%u", "8", '\x10'], + ["%d", "9", "%u", "9", '\x12'], + ["%d", "10", "%u", "10", '\x14'], + ["%d", "127", "%u", "127", '\xfe\x01'], + ["%d", "-32768", "%u", "4294934528", '\xff\xff\x03'], + ["%d", "652344632", "%u", "652344632", '\xf0\xf4\x8f\xee\x04'], + ["%d", "18567", "%u", "18567", '\x8e\xa2\x02'], + ["%d", "-14", "%u", "4294967282", '\x1b'], + ["%d", "-2147483648", "%u", "2147483648", '\xff\xff\xff\xff\x0f'], + ["%ld", "-14", "%lu", "4294967282", '\x1b'], + ["%d", "2075650855", "%u", "2075650855", '\xce\xac\xbf\xbb\x0f'], + ["%lld", "5922204476835468009", "%llu", "5922204476835468009", '\xd2\xcb\x8c\x90\x86\xe6\xf2\xaf\xa4\x01'], + ["%lld", "-9223372036854775808", "%llu", "9223372036854775808", '\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01'], + ["%lld", "3273441488341945355", "%llu", "3273441488341945355", '\x96\xb0\xae\x9a\x96\xec\xcc\xed\x5a'], + ["%lld", "-9223372036854775807", "%llu", "9223372036854775809", '\xfd\xff\xff\xff\xff\xff\xff\xff\xff\x01'], +] + +export default IntDB; diff --git a/pw_tokenizer/ts/printf_decoder.ts b/pw_tokenizer/ts/printf_decoder.ts index 127d35888..7b0203031 100644 --- a/pw_tokenizer/ts/printf_decoder.ts +++ b/pw_tokenizer/ts/printf_decoder.ts @@ -13,9 +13,9 @@ // the License. /** Decodes arguments and formats them with the provided format string. */ +import Long from "long"; -const SPECIFIER_REGEX = /%(\.([0-9]+))?([%csdioxXufFeEaAgGnp])/g; - +const SPECIFIER_REGEX = /%(\.([0-9]+))?(hh|h|ll|l|j|z|t|L)?([%csdioxXufFeEaAgGnp])/g; // Conversion specifiers by type; n is not supported. const SIGNED_INT = 'di'.split(''); const UNSIGNED_INT = 'oxXup'.split(''); @@ -33,14 +33,24 @@ enum DecodedStatusFlags { interface DecodedArg { size: number; - value: string | number | null; + value: string | number | Long | null; } // ZigZag decode function from protobuf's wire_format module. -function zigzagDecode(value: number): number { - if (!(value & 0x1)) return value >> 1; - return (value >> 1) ^ ~0; -} +function zigzagDecode(value: Long, unsigned: boolean = false): Long { + // 64 bit math is: + // signmask = (zigzag & 1) ? -1 : 0; + // twosComplement = (zigzag >> 1) ^ signmask; + // + // To work with 32 bit, we can operate on both but "carry" the lowest bit + // from the high word by shifting it up 31 bits to be the most significant bit + // of the low word. + var bitsLow = value.low, bitsHigh = value.high; + var signFlipMask = -(bitsLow & 1); + bitsLow = ((bitsLow >>> 1) | (bitsHigh << 31)) ^ signFlipMask; + bitsHigh = (bitsHigh >>> 1) ^ signFlipMask; + return new Long(bitsLow, bitsHigh, unsigned); +}; export class PrintfDecoder { // Reads a unicode string from the encoded data. @@ -65,16 +75,19 @@ export class PrintfDecoder { } private decodeSignedInt(args: Uint8Array): DecodedArg { - if (args.length === 0) return {size: 0, value: null}; + return this._decodeInt(args); + } + private _decodeInt(args: Uint8Array, unsigned: boolean = false): DecodedArg { + if (args.length === 0) return {size: 0, value: null}; let count = 0; - let result = 0; + let result = new Long(0); let shift = 0; for (count = 0; count < args.length; count++) { const byte = args[count]; - result |= (byte & 0x7f) << shift; + result = result.or((Long.fromInt(byte, unsigned).and(0x7f)).shiftLeft(shift)); if (!(byte & 0x80)) { - return {value: zigzagDecode(result), size: count + 1}; + return {value: zigzagDecode(result, unsigned), size: count + 1}; } shift += 7; if (shift >= 64) break; @@ -83,15 +96,21 @@ export class PrintfDecoder { return {size: 0, value: null}; } - private decodeUnsignedInt(args: Uint8Array): DecodedArg { - const arg = this.decodeSignedInt(args); + private decodeUnsignedInt(args: Uint8Array, lengthSpecifier: string): DecodedArg { + const arg = this._decodeInt(args, true); + const bits = ['ll', 'j'].indexOf(lengthSpecifier) !== -1 ? 64 : 32; // Since ZigZag encoding is used, unsigned integers must be masked off to // their original bit length. if (arg.value !== null) { - let num = arg.value as number; - num = num >>> 0; - arg.value = num; + let num = arg.value as Long; + if (bits === 32) { + num = num.and((Long.fromInt(1).shiftLeft(bits)).add(-1)); + } + else { + num = num.and(-1); + } + arg.value = num.toString(); } return arg; } @@ -100,8 +119,8 @@ export class PrintfDecoder { const arg = this.decodeSignedInt(args); if (arg.value !== null) { - const num = arg.value as number; - arg.value = String.fromCharCode(num); + const num = arg.value as Long; + arg.value = String.fromCharCode(num.toInt()); } return arg; } @@ -116,7 +135,7 @@ export class PrintfDecoder { return {size: 4, value: floatValue}; } - private format(specifierType: string, args: Uint8Array, precision: string): DecodedArg { + private format(specifierType: string, args: Uint8Array, precision: string, lengthSpecifier: string): DecodedArg { if (specifierType == '%') return {size: 0, value: '%'}; // literal % if (specifierType === 's') { return this.decodeString(args); @@ -128,7 +147,7 @@ export class PrintfDecoder { return this.decodeSignedInt(args); } if (UNSIGNED_INT.indexOf(specifierType) !== -1) { - return this.decodeUnsignedInt(args); + return this.decodeUnsignedInt(args, lengthSpecifier); } if (FLOATING_POINT.indexOf(specifierType) !== -1) { return this.decodeFloat(args, precision); @@ -141,11 +160,11 @@ export class PrintfDecoder { decode(formatString: string, args: Uint8Array): string { return formatString.replace( SPECIFIER_REGEX, - (_specifier, _precisionFull, precision, specifierType) => { - const decodedArg = this.format(specifierType, args, precision); - args = args.slice(decodedArg.size); - if (decodedArg === null) return ''; - return String(decodedArg.value); - }); + (_specifier, _precisionFull, precision, lengthSpecifier, specifierType) => { + const decodedArg = this.format(specifierType, args, precision, lengthSpecifier); + args = args.slice(decodedArg.size); + if (decodedArg === null) return ''; + return String(decodedArg.value); + }); } } diff --git a/pw_tokenizer/ts/printf_decoder_test.ts b/pw_tokenizer/ts/printf_decoder_test.ts index 80814ff76..db28e488a 100644 --- a/pw_tokenizer/ts/printf_decoder_test.ts +++ b/pw_tokenizer/ts/printf_decoder_test.ts @@ -14,6 +14,7 @@ /* eslint-env browser */ import {PrintfDecoder} from './printf_decoder'; +import IntDB from './int_testdata'; function argFromString(arg: string): Uint8Array { const data = new TextEncoder().encode(arg); @@ -52,6 +53,41 @@ describe('PrintfDecoder', () => { ).toEqual('Hello Mac and PC'); }); + it('formats string + number correctly', () => { + expect( + printfDecoder.decode( + 'Hello %s and %u', + argsConcat(argFromString('Computer'), argFromStringBinary('\xff\xff\x03')) + )).toEqual( + 'Hello Computer and 4294934528'); + }); + + it('formats integers correctly', () => { + for (let index = 0; index < IntDB.length; index++) { + const testcase = IntDB[index]; + // Test signed + expect( + printfDecoder + .decode(testcase[0], argFromStringBinary(testcase[4]))) + .toEqual(testcase[1]); + + // Test unsigned + expect( + printfDecoder + .decode(testcase[2], argFromStringBinary(testcase[4]))) + .toEqual(testcase[3]); + } + }); + + it('formats string correctly', () => { + expect( + printfDecoder.decode( + 'Hello %s and %s', + argsConcat(argFromString('Mac'), argFromString('PC')) + ) + ).toEqual('Hello Mac and PC'); + }); + it('formats varint correctly', () => { const arg = argFromStringBinary('\xff\xff\x03'); expect(printfDecoder.decode('Number %d', arg)).toEqual('Number -32768'); diff --git a/pw_toolchain/arm_clang/BUILD.gn b/pw_toolchain/arm_clang/BUILD.gn index 0e8cdaf14..966014284 100644 --- a/pw_toolchain/arm_clang/BUILD.gn +++ b/pw_toolchain/arm_clang/BUILD.gn @@ -40,6 +40,12 @@ cortex_m_hardware_fpu_v5_flags = cortex_m_hardware_fpu_v5_sp_flags = cortex_m_hardware_fpu_flags_common + [ "-mfpu=fpv5-sp-d16" ] +# Default config added to all the ARM cortex M targets to link `nosys` library. +config("nosys") { + # TODO(prabhukr): libs = ["nosys"] did not work as expected (pwrev/133110). + ldflags = [ "-lnosys" ] +} + config("enable_float_printf") { ldflags = [ "-Wl,-u_printf_float" ] } diff --git a/pw_toolchain/arm_clang/toolchains.gni b/pw_toolchain/arm_clang/toolchains.gni index 2b893c8e5..402082159 100644 --- a/pw_toolchain/arm_clang/toolchains.gni +++ b/pw_toolchain/arm_clang/toolchains.gni @@ -29,21 +29,45 @@ _arm_clang_toolchain = { } # Configs specific to different architectures. -_cortex_m0plus = [ "$dir_pw_toolchain/arm_clang:cortex_m0plus" ] +_cortex_m0plus = [ + "$dir_pw_toolchain/arm_clang:nosys", + "$dir_pw_toolchain/arm_clang:cortex_m0plus", +] -_cortex_m3 = [ "$dir_pw_toolchain/arm_clang:cortex_m3" ] +_cortex_m3 = [ + "$dir_pw_toolchain/arm_clang:nosys", + "$dir_pw_toolchain/arm_clang:cortex_m3", +] -_cortex_m4 = [ "$dir_pw_toolchain/arm_clang:cortex_m4" ] +_cortex_m4 = [ + "$dir_pw_toolchain/arm_clang:nosys", + "$dir_pw_toolchain/arm_clang:cortex_m4", +] -_cortex_m4f = [ "$dir_pw_toolchain/arm_clang:cortex_m4f" ] +_cortex_m4f = [ + "$dir_pw_toolchain/arm_clang:nosys", + "$dir_pw_toolchain/arm_clang:cortex_m4f", +] -_cortex_m7 = [ "$dir_pw_toolchain/arm_clang:cortex_m7" ] +_cortex_m7 = [ + "$dir_pw_toolchain/arm_clang:nosys", + "$dir_pw_toolchain/arm_clang:cortex_m7", +] -_cortex_m7f = [ "$dir_pw_toolchain/arm_clang:cortex_m7f" ] +_cortex_m7f = [ + "$dir_pw_toolchain/arm_clang:nosys", + "$dir_pw_toolchain/arm_clang:cortex_m7f", +] -_cortex_m33 = [ "$dir_pw_toolchain/arm_clang:cortex_m33" ] +_cortex_m33 = [ + "$dir_pw_toolchain/arm_clang:nosys", + "$dir_pw_toolchain/arm_clang:cortex_m33", +] -_cortex_m33f = [ "$dir_pw_toolchain/arm_clang:cortex_m33f" ] +_cortex_m33f = [ + "$dir_pw_toolchain/arm_clang:nosys", + "$dir_pw_toolchain/arm_clang:cortex_m33f", +] # Describes ARM clang toolchains for specific targets. pw_toolchain_arm_clang = { diff --git a/pw_toolchain/host_clang/toolchains.gni b/pw_toolchain/host_clang/toolchains.gni index e32b3afba..3e847c4af 100644 --- a/pw_toolchain/host_clang/toolchains.gni +++ b/pw_toolchain/host_clang/toolchains.gni @@ -28,6 +28,10 @@ declare_args() { # of the test binary itself cannot generate coverage reports. pw_toolchain_COVERAGE_ENABLED = false + # Indicates if this toolchain supports building fuzzers. This is typically + # set by individual toolchains and not by GN args. + pw_toolchain_FUZZING_ENABLED = false + # Indicates if this build is a part of OSS-Fuzz, which needs to be able to # provide its own compiler and flags. This violates the build hermeticisim and # should only be used for OSS-Fuzz. @@ -37,6 +41,9 @@ declare_args() { # Specifies the tools used by host Clang toolchains. _host_clang_toolchain = { if (pw_toolchain_OSS_FUZZ_ENABLED) { + # OSS-Fuzz sets compiler and linker paths. See + # google.github.io/oss-fuzz/getting-started/new-project-guide/#Requirements. + # Just use the "llvm-ar" on the system path. ar = "llvm-ar" cc = getenv("CC") @@ -127,6 +134,9 @@ pw_toolchain_host_clang = { defaults = { forward_variables_from(_defaults, "*") + pw_toolchain_FUZZING_ENABLED = true + default_configs += [ "$dir_pw_fuzzer:instrumentation" ] + # Always disable coverage generation. pw_toolchain_COVERAGE_ENABLED = false @@ -141,10 +151,6 @@ pw_toolchain_host_clang = { default_configs += [ "$dir_pw_toolchain/host_clang:sanitize_$sanitizer" ] } - - if (pw_toolchain_OSS_FUZZ_ENABLED) { - default_configs += [ "$dir_pw_fuzzer:oss_fuzz_extra" ] - } } } diff --git a/pw_toolchain/py/pw_toolchain/clang_arm_toolchain.py b/pw_toolchain/py/pw_toolchain/clang_arm_toolchain.py index d1ba0d389..5a02f3896 100644 --- a/pw_toolchain/py/pw_toolchain/clang_arm_toolchain.py +++ b/pw_toolchain/py/pw_toolchain/clang_arm_toolchain.py @@ -169,7 +169,6 @@ def get_crt_objs(compiler_info: Dict[str, str]) -> Tuple[str, ...]: def get_ldflags(compiler_info: Dict[str, str]) -> List[str]: ldflags: List[str] = [ - '-lnosys', # Add library search paths. '-L' + compiler_info['gcc_libs_dir'], '-L' diff --git a/pw_trace/docs.rst b/pw_trace/docs.rst index 7e328aa66..cfc46586f 100644 --- a/pw_trace/docs.rst +++ b/pw_trace/docs.rst @@ -199,14 +199,23 @@ Currently the included python tool supports a few different options for can be used to either provide a single value type, or provide multiple different values with a variety of types. Options for format string types can be found here: https://docs.python.org/3/library/struct.html#format-characters + . The data is always assumed to be packed with little-endian ordering if not + indicated otherwise:: + + // Example + data_format_string = "@pw_py_struct_fmt:ll" + data = 0x1400000014000000 + args = {data_0: 20, data_1:20} - *@pw_py_map_fmt:* - Interprets the string after the ":" as a dictionary relating the data field name to the python struct format string. Once collected, the format strings are concatenated and used to unpack the data - elements as above:: + elements as above. The data is always assumed to be packed with little-endian + ordering if not indicated otherwise. To specify a different ordering, + construct the format string as ``@pw_py_map_fmt:[@=<>!]{k:v,...}``:: // Example data_format_string = "@pw_py_map_fmt:{Field: l, Field2: l }" - data = 0x14000000000000001400000000000000 (little endian) + data = 0x1400000014000000 args = {Field: 20, Field2:20} .. tip:: diff --git a/pw_trace/public/pw_trace/internal/trace_internal.h b/pw_trace/public/pw_trace/internal/trace_internal.h index fc106b80c..982765f99 100644 --- a/pw_trace/public/pw_trace/internal/trace_internal.h +++ b/pw_trace/public/pw_trace/internal/trace_internal.h @@ -242,8 +242,10 @@ static inline void _pw_trace_disabled(int x, ...) { (void)x; } object_name(object_name&&) = delete; \ object_name& operator=(const object_name&) = delete; \ object_name& operator=(object_name&&) = delete; \ - object_name(uint32_t trace_id = PW_TRACE_TRACE_ID_DEFAULT) \ - : trace_id_(trace_id) { \ + \ + object_name(uint32_t PW_CONCAT(object_name, \ + _trace_id) = PW_TRACE_TRACE_ID_DEFAULT) \ + : trace_id_(PW_CONCAT(object_name, _trace_id)) { \ _PW_TRACE_IF_ENABLED(event_type_start, flag, label, group, trace_id_); \ } \ ~object_name() { \ diff --git a/pw_trace/py/pw_trace/trace.py b/pw_trace/py/pw_trace/trace.py index dbd2b7d16..571d66f0c 100755 --- a/pw_trace/py/pw_trace/trace.py +++ b/pw_trace/py/pw_trace/trace.py @@ -28,6 +28,7 @@ import struct from typing import Iterable, NamedTuple _LOG = logging.getLogger('pw_trace') +_ORDERING_CHARS = ("@", "=", "<", ">", "!") class TraceType(Enum): @@ -69,17 +70,47 @@ def event_has_trace_id(event_type): } +def decode_struct_fmt_args(event): + """Decodes the trace's event data for struct-formatted data""" + args = {} + # we assume all data is packed, little-endian ordering if not specified + struct_fmt = event.data_fmt[len("@pw_py_struct_fmt:") :] + if not struct_fmt.startswith(_ORDERING_CHARS): + struct_fmt = "<" + struct_fmt + try: + # needed in case the buffer is larger than expected + assert struct.calcsize(struct_fmt) == len(event.data) + items = struct.unpack_from(struct_fmt, event.data) + for i, item in enumerate(items): + args["data_" + str(i)] = item + except (AssertionError, struct.error): + args["error"] = ( + f"Mismatched struct/data format {event.data_fmt} " + f"expected data len {struct.calcsize(struct_fmt)} " + f"data {event.data.hex()} " + f"data len {len(event.data)}" + ) + return args + + def decode_map_fmt_args(event): """Decodes the trace's event data for map-formatted data""" args = {} - fmt_list = event.data_fmt[len("@pw_py_map_fmt:") :].strip("{}").split(",") - fmt_bytes = '' - fields = [] + fmt = event.data_fmt[len("@pw_py_map_fmt:") :] + + # we assume all data is packed, little-endian ordering if not specified + if not fmt.startswith(_ORDERING_CHARS): + fmt = '<' + fmt + try: + (fmt_bytes, fmt_list) = fmt.split("{") + fmt_list = fmt_list.strip("}").split(",") + + names = [] for pair in fmt_list: - (field, value) = (s.strip() for s in pair.split(":")) - fields.append(field) - fmt_bytes += value + (name, fmt_char) = (s.strip() for s in pair.split(":")) + names.append(name) + fmt_bytes += fmt_char except ValueError: args["error"] = f"Invalid map format {event.data_fmt}" else: @@ -88,11 +119,13 @@ def decode_map_fmt_args(event): assert struct.calcsize(fmt_bytes) == len(event.data) items = struct.unpack_from(fmt_bytes, event.data) for i, item in enumerate(items): - args[fields[i]] = item + args[names[i]] = item except (AssertionError, struct.error): args["error"] = ( - f"Mismatched struct/data format {event.data_fmt}" - f" data {event.data.hex()}" + f"Mismatched map/data format {event.data_fmt} " + f"expected data len {struct.calcsize(fmt_bytes)} " + f"data {event.data.hex()} " + f"data len {len(event.data)}" ) return args @@ -171,13 +204,7 @@ def generate_trace_json(events: Iterable[TraceEvent]): line["name"]: int.from_bytes(event.data, "little") } elif event.data_fmt.startswith("@pw_py_struct_fmt:"): - items = struct.unpack_from( - event.data_fmt[len("@pw_py_struct_fmt:") :], event.data - ) - args = {} - for i, item in enumerate(items): - args["data_" + str(i)] = item - line["args"] = args + line["args"] = decode_struct_fmt_args(event) elif event.data_fmt.startswith("@pw_py_map_fmt:"): line["args"] = decode_map_fmt_args(event) else: diff --git a/pw_trace/py/trace_test.py b/pw_trace/py/trace_test.py index 9cec920bb..60249623d 100755 --- a/pw_trace/py/trace_test.py +++ b/pw_trace/py/trace_test.py @@ -195,7 +195,7 @@ class TestTraceGenerateJson(unittest.TestCase): timestamp_us=10, has_data=True, data_fmt="@pw_py_struct_fmt:Hl", - data=struct.pack("Hl", 5, 2), + data=struct.pack("<Hl", 5, 2), ) json_lines = trace.generate_trace_json([event]) self.assertEqual(1, len(json_lines)) @@ -211,6 +211,62 @@ class TestTraceGenerateJson(unittest.TestCase): }, ) + def test_generate_error_json_data_struct_invalid_small_buffer(self): + event = trace.TraceEvent( + event_type=trace.TraceType.INSTANTANEOUS, + module="module", + label="counter", + timestamp_us=10, + has_data=True, + data_fmt="@pw_py_struct_fmt:Hl", + data=struct.pack("<H", 5), + ) + json_lines = trace.generate_trace_json([event]) + self.assertEqual(1, len(json_lines)) + self.assertEqual( + json.loads(json_lines[0]), + { + "ph": "I", + "pid": "module", + "name": "counter", + "ts": 10, + "s": "p", + "args": { + "error": f"Mismatched struct/data format {event.data_fmt} " + f"expected data len {struct.calcsize('<Hl')} data " + f"{event.data.hex()} data len {len(event.data)}" + }, + }, + ) + + def test_generate_error_json_data_struct_invalid_large_buffer(self): + event = trace.TraceEvent( + event_type=trace.TraceType.INSTANTANEOUS, + module="module", + label="counter", + timestamp_us=10, + has_data=True, + data_fmt="@pw_py_struct_fmt:Hl", + data=struct.pack("<Hll", 5, 2, 5), + ) + json_lines = trace.generate_trace_json([event]) + self.assertEqual(1, len(json_lines)) + self.assertEqual( + json.loads(json_lines[0]), + { + "ph": "I", + "pid": "module", + "name": "counter", + "ts": 10, + "s": "p", + "args": { + "error": f"Mismatched struct/data format {event.data_fmt} " + f"expected data len {struct.calcsize('<Hl')} data " + f"{event.data.hex()} data len {len(event.data)}" + }, + }, + ) + def test_generate_json_data_map_fmt_single(self): event = trace.TraceEvent( event_type=trace.TraceType.INSTANTANEOUS, @@ -219,7 +275,7 @@ class TestTraceGenerateJson(unittest.TestCase): timestamp_us=10, has_data=True, data_fmt="@pw_py_map_fmt:{Field:l}", - data=struct.pack("l", 20), + data=struct.pack("<l", 20), ) json_lines = trace.generate_trace_json([event]) self.assertEqual(1, len(json_lines)) @@ -243,7 +299,7 @@ class TestTraceGenerateJson(unittest.TestCase): timestamp_us=10, has_data=True, data_fmt="@pw_py_map_fmt:{Field: l, Field2: l }", - data=struct.pack("ll", 20, 40), + data=struct.pack("<ll", 20, 40), ) json_lines = trace.generate_trace_json([event]) self.assertEqual(1, len(json_lines)) @@ -267,7 +323,7 @@ class TestTraceGenerateJson(unittest.TestCase): timestamp_us=10, has_data=True, data_fmt="@pw_py_map_fmt:{Field;l,Field2;l}", - data=struct.pack("ll", 20, 40), + data=struct.pack("<ll", 20, 40), ) json_lines = trace.generate_trace_json([event]) self.assertEqual(1, len(json_lines)) @@ -291,7 +347,7 @@ class TestTraceGenerateJson(unittest.TestCase): timestamp_us=10, has_data=True, data_fmt="@pw_py_map_fmt:{Field:l,Field2:l}", - data=struct.pack("l", 20), + data=struct.pack("<l", 20), ) json_lines = trace.generate_trace_json([event]) self.assertEqual(1, len(json_lines)) @@ -304,8 +360,9 @@ class TestTraceGenerateJson(unittest.TestCase): "ts": 10, "s": "p", "args": { - "error": f"Mismatched struct/data format {event.data_fmt} " - f"data {event.data.hex()}" + "error": f"Mismatched map/data format {event.data_fmt} " + f"expected data len {struct.calcsize('<ll')} data " + f"{event.data.hex()} data len {len(event.data)}" }, }, ) @@ -318,7 +375,7 @@ class TestTraceGenerateJson(unittest.TestCase): timestamp_us=10, has_data=True, data_fmt="@pw_py_map_fmt:{Field:H,Field2:H}", - data=struct.pack("ll", 20, 40), + data=struct.pack("<ll", 20, 40), ) json_lines = trace.generate_trace_json([event]) self.assertEqual(1, len(json_lines)) @@ -331,8 +388,9 @@ class TestTraceGenerateJson(unittest.TestCase): "ts": 10, "s": "p", "args": { - "error": f"Mismatched struct/data format {event.data_fmt} " - f"data {event.data.hex()}" + "error": f"Mismatched map/data format {event.data_fmt} " + f"expected data len {struct.calcsize('<HH')} data " + f"{event.data.hex()} data len {len(event.data)}" }, }, ) diff --git a/pw_trace_tokenized/py/pw_trace_tokenized/get_trace.py b/pw_trace_tokenized/py/pw_trace_tokenized/get_trace.py index 38056f37b..041c6bcdf 100755 --- a/pw_trace_tokenized/py/pw_trace_tokenized/get_trace.py +++ b/pw_trace_tokenized/py/pw_trace_tokenized/get_trace.py @@ -33,7 +33,7 @@ import socket import sys from typing import Collection, Iterable, Iterator -import serial # type: ignore +import serial from pw_tokenizer import database from pw_trace import trace from pw_hdlc.rpc import HdlcRpcClient, default_channels diff --git a/pw_trace_tokenized/py/setup.cfg b/pw_trace_tokenized/py/setup.cfg index 6c643bb4a..98038d397 100644 --- a/pw_trace_tokenized/py/setup.cfg +++ b/pw_trace_tokenized/py/setup.cfg @@ -22,6 +22,8 @@ description = pw_trace backend to tokenize trace events packages = find: zip_safe = False install_requires = + pyserial>=3.5,<4.0 + types-pyserial>=3.5,<4.0 [options.package_data] pw_trace_tokenized = py.typed diff --git a/pw_unit_test/py/pw_unit_test/test_runner.py b/pw_unit_test/py/pw_unit_test/test_runner.py index bc88f9918..a7b06ac5e 100644 --- a/pw_unit_test/py/pw_unit_test/test_runner.py +++ b/pw_unit_test/py/pw_unit_test/test_runner.py @@ -306,6 +306,7 @@ class TestRunner: % self._result_sink['auth_token'], }, data=json.dumps({'testResults': [test_result]}), + timeout=5.0, ).raise_for_status() diff --git a/pw_unit_test/test.gni b/pw_unit_test/test.gni index 7944a77bd..86399f7e0 100644 --- a/pw_unit_test/test.gni +++ b/pw_unit_test/test.gni @@ -219,7 +219,60 @@ template("pw_test") { pw_internal_disableable_target("$target_name.lib") { target_type = "pw_source_set" enable_if = _test_is_enabled - forward_variables_from(invoker, "*", [ "metadata" ]) + + # It is possible that the executable target type has been overriden by + # pw_unit_test_EXECUTABLE_TARGET_TYPE, which may allow for additional + # variables to be specified on the executable template. As such, we cannot + # forward all variables ("*") from the invoker to source_set library, as + # those additional variables would not be used and GN gen would error. + _source_set_relevant_variables = [ + # GN source_set variables + # https://gn.googlesource.com/gn/+/main/docs/reference.md#target-declarations-source_set_declare-a-source-set-target-variables + "asmflags", + "cflags", + "cflags_c", + "cflags_cc", + "cflags_objc", + "cflags_objcc", + "defines", + "include_dirs", + "inputs", + "ldflags", + "lib_dirs", + "libs", + "precompiled_header", + "precompiled_source", + "rustenv", + "rustflags", + "swiftflags", + "testonly", + "assert_no_deps", + "data_deps", + "deps", + "public_deps", + "runtime_deps", + "write_runtime_deps", + "all_dependent_configs", + "public_configs", + "check_includes", + "configs", + "data", + "friend", + "inputs", + "metadata", + "output_extension", + "output_name", + "public", + "sources", + "testonly", + "visibility", + + # pw_source_set variables + # https://pigweed.dev/pw_build/?highlight=pw_executable#target-types + "remove_configs", + "remove_public_deps", + ] + forward_variables_from(invoker, _source_set_relevant_variables) if (!defined(deps)) { deps = [] @@ -239,6 +292,17 @@ template("pw_test") { target_type = pw_unit_test_EXECUTABLE_TARGET_TYPE enable_if = _test_is_enabled + # Include configs, deps, etc. from the pw_test in the executable as well as + # the library to ensure that linker flags propagate to the executable. + forward_variables_from(invoker, + "*", + [ + "extra_metadata", + "metadata", + "sources", + "public", + ]) + # Metadata for this test when used as part of a pw_test_group target. metadata = { tests = [ @@ -263,7 +327,10 @@ template("pw_test") { } } - deps = [ ":$_test_target_name.lib" ] + if (!defined(deps)) { + deps = [] + } + deps += [ ":$_test_target_name.lib" ] if (_test_main != "") { deps += [ _test_main ] } diff --git a/pw_watch/py/pw_watch/debounce.py b/pw_watch/py/pw_watch/debounce.py index c3559a177..ce09063b0 100644 --- a/pw_watch/py/pw_watch/debounce.py +++ b/pw_watch/py/pw_watch/debounce.py @@ -103,14 +103,14 @@ class Debouncer: # re-try running afterwards. error_message = ['Event while running: %s', event_description] if BUILDER_CONTEXT.using_progress_bars(): - _LOG.error(*error_message) + _LOG.warning(*error_message) else: # Push an empty line to flush ongoing I/O in subprocess. print('') # Surround the error message with newlines to make it stand out. print('') - _LOG.error(*error_message) + _LOG.warning(*error_message) print('') self.function.cancel() diff --git a/pw_watch/py/pw_watch/watch.py b/pw_watch/py/pw_watch/watch.py index 0379a39cb..d956df3ec 100755 --- a/pw_watch/py/pw_watch/watch.py +++ b/pw_watch/py/pw_watch/watch.py @@ -252,7 +252,7 @@ class PigweedBuildWatcher(FileSystemEventHandler, DebouncedFunction): log_message = f'File change detected: {os.path.relpath(matching_path)}' if self.restart_on_changes: if self.fullscreen_enabled and self.watch_app: - self.watch_app.rebuild_on_filechange() + self.watch_app.clear_log_panes() self.debouncer.press(f'{log_message} Triggering build...') else: _LOG.info('%s ; not rebuilding', log_message) @@ -340,6 +340,11 @@ class PigweedBuildWatcher(FileSystemEventHandler, DebouncedFunction): BUILDER_CONTEXT.set_idle() def run_recipe(self, index: int, cfg: BuildRecipe, env) -> None: + if BUILDER_CONTEXT.interrupted(): + return + if not cfg.enabled: + return + num_builds = len(self.project_builder) index_message = f'[{index}/{num_builds}]' @@ -438,7 +443,11 @@ class PigweedBuildWatcher(FileSystemEventHandler, DebouncedFunction): _LOG.info('Build stopped.') elif BUILDER_CONTEXT.interrupted(): pass # Don't print anything. - elif all(recipe.status.passed() for recipe in self.project_builder): + elif all( + recipe.status.passed() + for recipe in self.project_builder + if recipe.enabled + ): _LOG.info('Finished; all successful') else: _LOG.info('Finished; some builds failed') diff --git a/pw_watch/py/pw_watch/watch_app.py b/pw_watch/py/pw_watch/watch_app.py index df4dbf6c1..38514533d 100644 --- a/pw_watch/py/pw_watch/watch_app.py +++ b/pw_watch/py/pw_watch/watch_app.py @@ -15,6 +15,7 @@ """ Prompt toolkit application for pw watch. """ import asyncio +import functools import logging import os import re @@ -45,13 +46,14 @@ from prompt_toolkit.layout import ( ) from prompt_toolkit.layout.controls import BufferControl from prompt_toolkit.styles import ( + ConditionalStyleTransformation, DynamicStyle, + SwapLightAndDarkStyleTransformation, + merge_style_transformations, merge_styles, - Style, style_from_pygments_cls, ) from prompt_toolkit.formatted_text import StyleAndTextTuples -from prompt_toolkit.mouse_events import MouseEvent, MouseEventType from prompt_toolkit.lexers import PygmentsLexer from pygments.lexers.markup import MarkdownLexer # type: ignore @@ -70,6 +72,8 @@ from pw_console.widgets import ( ToolbarButton, WindowPaneToolbar, create_border, + mouse_handlers, + to_checkbox, ) from pw_console.window_list import DisplayMode from pw_console.window_manager import WindowManager @@ -112,6 +116,33 @@ Move window pane up. ------------------------------ Ctrl-Alt-Up Balance all window sizes. ------------------------- Ctrl-U +Bottom Toolbar Controls +======================= + +Rebuild Enter --------------- Click or press Enter to trigger a rebuild. +[x] Auto Rebuild ------------ Click to globaly enable or disable automatic + rebuilding when files change. +Help F1 --------------------- Click or press F1 to open this help window. +Quit Ctrl-d ----------------- Click or press Ctrl-d to quit pw_watch. +Next Tab Ctrl-Alt-n --------- Switch to the next log tab. +Previous Tab Ctrl-Alt-p ----- Switch to the previous log tab. + + +Build Status Bar +================ + +The build status bar shows the current status of all build directories outlined +in a colored frame. + + ┏━━ BUILDING ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ + ┃ [✓] out_directory Building Last line of standard out. ┃ + ┃ [✓] out_dir2 Waiting Last line of standard out. ┃ + ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +Each checkbox on the far left controls whether that directory is built when +files change and manual builds are run. + + Copying Text ============ @@ -163,13 +194,12 @@ class WatchAppPrefs(ProjectBuilderPrefs): self.registered_commands = DEFAULT_KEY_BINDINGS self.registered_commands.update(self.user_key_bindings) - self.default_config.update( - { - 'key_bindings': DEFAULT_KEY_BINDINGS, - 'show_python_logger': True, - } - ) - self.reset_config() + new_config_settings = { + 'key_bindings': DEFAULT_KEY_BINDINGS, + 'show_python_logger': True, + } + self.default_config.update(new_config_settings) + self._update_config(new_config_settings) # Required pw_console preferences for key bindings and themes @property @@ -188,6 +218,10 @@ class WatchAppPrefs(ProjectBuilderPrefs): def theme_colors(self): return get_theme_colors(self.ui_theme) + @property + def swap_light_and_dark(self) -> bool: + return self._config.get('swap_light_and_dark', False) + def get_function_keys(self, name: str) -> List: """Return the keys for the named function.""" try: @@ -256,38 +290,6 @@ class WatchWindowManager(WindowManager): self.application.window_manager_container = self.create_root_container() -class StatusBarControl(FormattedTextControl): - """Handles switching build tabs in the UI on mouse click.""" - - def __init__(self, watch_app: 'WatchApp', *args, **kwargs) -> None: - self.watch_app = watch_app - super().__init__(*args, **kwargs) - - def mouse_handler(self, mouse_event: MouseEvent): - """Mouse handler for this control.""" - _click_x = mouse_event.position.x - _click_y = mouse_event.position.y - - # On left click - if mouse_event.event_type == MouseEventType.MOUSE_UP: - tab_index = _click_y - pane = self.watch_app.recipe_index_to_log_pane.get(tab_index, None) - if not pane: - return NotImplemented - - ( - window_list, - pane_index, - ) = self.watch_app.window_manager.find_window_list_and_pane_index( - pane - ) - window_list.switch_to_tab(pane_index) - return None - - # Mouse event not handled, return NotImplemented. - return NotImplemented - - class WatchApp(PluginMixin): """Pigweed Watch main window application.""" @@ -371,9 +373,7 @@ class WatchApp(PluginMixin): self.status_bar_border_style = 'class:command-runner-border' - self.status_bar_control = StatusBarControl( - self, self.get_status_bar_text - ) + self.status_bar_control = FormattedTextControl(self.get_status_bar_text) self.status_bar_container = create_border( HSplit( @@ -413,6 +413,17 @@ class WatchApp(PluginMixin): click_to_focus_text='', ) self.help_toolbar.add_button( + ToolbarButton('Enter', 'Rebuild', self.run_build) + ) + self.help_toolbar.add_button( + ToolbarButton( + description='Auto Rebuild', + mouse_handler=self.toggle_restart_on_filechange, + is_checkbox=True, + checked=lambda: self.restart_on_changes, + ) + ) + self.help_toolbar.add_button( ToolbarButton('F1', 'Help', self.user_guide_window.toggle_display) ) self.help_toolbar.add_button(ToolbarButton('Ctrl-d', 'Quit', self.exit)) @@ -491,11 +502,16 @@ class WatchApp(PluginMixin): ) self.current_theme = generate_styles(self.prefs.ui_theme) - self.style_overrides = Style.from_dict( - { - # 'search': 'bg:ansired ansiblack', - } + + self.style_transformation = merge_style_transformations( + [ + ConditionalStyleTransformation( + SwapLightAndDarkStyleTransformation(), + filter=Condition(lambda: self.prefs.swap_light_and_dark), + ), + ] ) + self.code_theme = style_from_pygments_cls(PigweedCodeStyle) self.layout = Layout( @@ -513,11 +529,11 @@ class WatchApp(PluginMixin): lambda: merge_styles( [ self.current_theme, - self.style_overrides, self.code_theme, ] ) ), + style_transformation=self.style_transformation, full_screen=True, ) @@ -645,7 +661,7 @@ class WatchApp(PluginMixin): instead.""" self.window_manager.focus_first_visible_pane() - def switch_to_root_log(self): + def switch_to_root_log(self) -> None: ( window_list, pane_index, @@ -654,6 +670,17 @@ class WatchApp(PluginMixin): ) window_list.switch_to_tab(pane_index) + def switch_to_build_log(self, log_index: int) -> None: + pane = self.recipe_index_to_log_pane.get(log_index, None) + if not pane: + return + + ( + window_list, + pane_index, + ) = self.window_manager.find_window_list_and_pane_index(pane) + window_list.switch_to_tab(pane_index) + def command_runner_is_open(self) -> bool: # pylint: disable=no-self-use return False @@ -663,25 +690,34 @@ class WatchApp(PluginMixin): if isinstance(pane, LogPane): yield pane - def clear_ninja_log(self) -> None: + def clear_log_panes(self) -> None: + """Erase all log pane content and turn on follow. + + This is called whenever rebuilds occur. Either a manual build from + self.run_build or on file changes called from + pw_watch._handle_matched_event.""" for pane in self.all_log_panes(): + pane.log_view.clear_visual_selection() + pane.log_view.clear_filters() pane.log_view.log_store.clear_logs() - pane.log_view._restart_filtering() # pylint: disable=protected-access pane.log_view.view_mode_changed() # Re-enable follow if needed if not pane.log_view.follow: pane.log_view.toggle_follow() - def run_build(self): - """Manually trigger a rebuild.""" - self.clear_ninja_log() + def run_build(self) -> None: + """Manually trigger a rebuild from the UI.""" + self.clear_log_panes() self.event_handler.rebuild() - def rebuild_on_filechange(self): - for pane in self.all_log_panes(): - pane.log_view.clear_visual_selection() - pane.log_view.log_store.clear_logs() - pane.log_view.view_mode_changed() + @property + def restart_on_changes(self) -> bool: + return self.event_handler.restart_on_changes + + def toggle_restart_on_filechange(self) -> None: + self.event_handler.restart_on_changes = ( + not self.event_handler.restart_on_changes + ) def get_status_bar_text(self) -> StyleAndTextTuples: """Return formatted text for build status bar.""" @@ -696,21 +732,48 @@ class WatchApp(PluginMixin): pane, ) = self.window_manager._get_active_window_list_and_pane() # pylint: enable=protected-access + restarting = BUILDER_CONTEXT.restart_flag - for cfg in self.event_handler.project_builder: + for i, cfg in enumerate(self.event_handler.project_builder): # The build directory name_style = '' - if pane and pane.pane_title() == cfg.display_name: + if not pane: + formatted_text.append(('', '\n')) + continue + + # Dim the build name if disabled + if not cfg.enabled: + name_style = 'class:theme-fg-inactive' + + # If this build tab is selected, highlight with cyan. + if pane.pane_title() == cfg.display_name: name_style = 'class:theme-fg-cyan' + + formatted_text.append( + to_checkbox( + cfg.enabled, + functools.partial( + mouse_handlers.on_click, + cfg.toggle_enabled, + ), + end=' ', + unchecked_style='class:checkbox', + checked_style='class:checkbox-checked', + ) + ) formatted_text.append( ( name_style, f'{cfg.display_name}'.ljust(name_width), + functools.partial( + mouse_handlers.on_click, + functools.partial(self.switch_to_build_log, i), + ), ) ) formatted_text.append(separator) # Status - formatted_text.append(cfg.status.status_slug()) + formatted_text.append(cfg.status.status_slug(restarting=restarting)) formatted_text.append(separator) # Current stdout line formatted_text.extend(cfg.status.current_step_formatted()) @@ -724,13 +787,15 @@ class WatchApp(PluginMixin): return formatted_text def set_tab_bar_colors(self) -> None: + restarting = BUILDER_CONTEXT.restart_flag + for cfg in BUILDER_CONTEXT.recipes: pane = self.recipe_name_to_log_pane.get(cfg.display_name, None) if not pane: continue pane.extra_tab_style = None - if cfg.status.failed(): + if not restarting and cfg.status.failed(): pane.extra_tab_style = 'class:theme-fg-red' def exit( diff --git a/seed/0000-index.rst b/seed/0000-index.rst index 142add415..35f1e753f 100644 --- a/seed/0000-index.rst +++ b/seed/0000-index.rst @@ -10,5 +10,5 @@ All pending, active, and resolved SEEDs are listed below. 0001-the-seed-process 0002-template - 0101: pw_project.toml<https://pigweed-review.git.corp.google.com/c/pigweed/pigweed/+/128010> + 0101-pigweed.json 0102-module-docs diff --git a/seed/0001-the-seed-process.rst b/seed/0001-the-seed-process.rst index d697b163d..3b7958bd6 100644 --- a/seed/0001-the-seed-process.rst +++ b/seed/0001-the-seed-process.rst @@ -8,11 +8,11 @@ :fas:`seedling` SEED-0001: :ref:`The SEED Process<seed-0001>` :octicon:`comment-discussion` Status: - :bdg-primary:`Open for Comments` + :bdg-secondary-line:`Open for Comments` :octicon:`chevron-right` :bdg-secondary-line:`Last Call` :octicon:`chevron-right` - :bdg-secondary-line:`Accepted` + :bdg-primary:`Accepted` :octicon:`kebab-horizontal` :bdg-secondary-line:`Rejected` diff --git a/seed/0101-pigweed.json.rst b/seed/0101-pigweed.json.rst new file mode 100644 index 000000000..b8def7f82 --- /dev/null +++ b/seed/0101-pigweed.json.rst @@ -0,0 +1,210 @@ +.. _seed-0101: + +================== +0101: pigweed.json +================== + +.. card:: + :fas:`seedling` SEED-0101: :ref:`pigweed.json<seed-0101>` + + :octicon:`comment-discussion` Status: + :bdg-secondary-line:`Open for Comments` + :octicon:`chevron-right` + :bdg-secondary-line:`Last Call` + :octicon:`chevron-right` + :bdg-primary:`Accepted` + :octicon:`kebab-horizontal` + :bdg-secondary-line:`Rejected` + + :octicon:`calendar` Proposal Date: 2023-02-06 + + :octicon:`code-review` CL: `pwrev/128010 <https://pigweed-review.git.corp.google.com/c/pigweed/pigweed/+/128010>`_ + +------- +Summary +------- +Combine several of the configuration options downstream projects use to +configure parts of Pigweed in one place, and use this place for further +configuration options. + +---------- +Motivation +---------- +Pigweed-based projects configure Pigweed and themselves in a variety of ways. +The environment setup is controlled by a JSON file that's referenced in +``bootstrap.sh`` files and in internal infrastructure repos that looks +something like this: + +.. code-block:: + + { + "root_variable": "<PROJNAME>_ROOT", + "cipd_package_files": ["tools/default.json"], + "virtualenv": { + "gn_args": ["dir_pw_third_party_stm32cube=\"\""], + "gn_root": ".", + "gn_targets": [":python.install"] + }, + "optional_submodules": ["vendor/shhh-secret"], + "gni_file": "build_overrides/pigweed_environment.gni" + } + +The plugins to the ``pw`` command-line utility are configured in ``PW_PLUGINS``, +which looks like this: + +.. code-block:: + + # <name> <Python module> <function> + console pw_console.__main__ main + format pw_presubmit.format_code _pigweed_upstream_main + +In addition, changes have been proposed to configure some of the behavior of +``pw format`` and the formatting steps of ``pw presubmit`` from config files, +but there's no standard place to put these configuration options. + +--------------- +Guide reference +--------------- +This proposal affects two sets of people: developers looking to use Pigweed, +and developers looking to add configurable features to Pigweed. + +Developers looking to use Pigweed will have one config file that contains all +the options they need to set. Documentation for individual Pigweed modules will +show only the configuration options relevant for that module, and multiple of +these examples can simply be concatenated to form a valid config file. + +Developers looking to add configurable features to Pigweed no longer need to +define a new file format, figure out where to find it in the tree (or how to +have Pigweed-projects specify a location), or parse this format. + +--------------------- +Problem investigation +--------------------- +There are multiple issues with the current system that need to be addressed. + +* ``PW_PLUGINS`` works, but is a narrow custom format with exactly one purpose. +* The environment config file is somewhat extensible, but is still specific to + environment setup. +* There's no accepted place for other modules to retrieve configuration options. + +These should be combined into a single file. There are several formats that +could be selected, and many more arguments for and against each. Only a subset +of these arguments are reproduced here. + +* JSON does not support comments +* JSON5 is not supported in the Python standard library +* XML is too verbose +* YAML is acceptable, but implicit type conversion could be a problem, and it's + not supported in the Python standard library +* TOML is acceptable, and `was selected for a similar purpose by Python + <https://snarky.ca/what-the-heck-is-pyproject-toml/>`_, but it's + not supported in the Python standard library before Python v3.11 +* Protobuf Text Format is acceptable and widely used within Google, but is not + supported in the Python standard library + +The location of the file is also an issue. Environment config files can be found +in a variety of locations depending on the project—all of the following paths +are used by at least one internal Pigweed-based project. + +* ``build/environment.json`` +* ``build/pigweed/env_setup.json`` +* ``environment.json`` +* ``env_setup.json`` +* ``pw_env_setup.json`` +* ``scripts/environment.json`` +* ``tools/environment.json`` +* ``tools/env_setup.json`` + +``PW_PLUGINS`` files can in theory be in any directory and ``pw`` will search up +for them from the current directory, but in practice they only exist at the root +of checkouts. Having this file in a fixed location with a fixed name makes it +significantly easier to find as a user, and the fixed name (if not path) makes +it easy to find programmatically too. + +--------------- +Detailed design +--------------- +The ``pw_env_setup`` Python module will provide an API to retrieve a parsed +``pigweed.json`` file from the root of the checkout. ``pw_env_setup`` is the +correct location because it can't depend on anything else, but other modules can +depend on it. Code in other languages does not yet depend on configuration +files. + +A ``pigweed.json`` file might look like the following. Individual option names +and structures are not final but will evolve as those options are +implemented—this is merely an example of what an actual file could look like. +The ``pw`` namespace is reserved for Pigweed, but other projects can use other +namespaces for their own needs. Within the ``pw`` namespace all options are +first grouped by their module name, which simplifies searching for the code and +documentation related to the option in question. + +.. code-block:: + + { + "pw": { + "pw_cli": { + "plugins": { + "console": { + "module": "pw_console.__main__", + "function": "main" + }, + "format": { + "module": "pw_presubmit.format_code", + "function": "_pigweed_upstream_main" + } + } + }, + "pw_env_setup": { + "root_variable": "<PROJNAME>_ROOT", + "rosetta": "allow", + "gni_file": "build_overrides/pigweed_environment.gni", + "cipd": { + "package_files": [ + "tools/default.json" + ] + }, + "virtualenv": { + "gn_args": [ + "dir_pw_third_party_stm32cube=\"\"" + ], + "gn_targets": [ + "python.install" + ], + "gn_root": "." + }, + "submodules": { + "optional": [ + "vendor/shhh-secret" + ] + } + }, + "pw_presubmit": { + "format": { + "python": { + "formatter": "black", + "black_path": "pyink" + } + } + } + } + } + +Some teams will resist a new file at the root of their checkout, but this seed +won't be adding any files, it'll be combining at least one top-level file, maybe +two, into a new top-level file, so there won't be any additional files in the +checkout root. + +------------ +Alternatives +------------ +``pw format`` and the formatting steps of ``pw presubmit`` could read from yet +another config file, further fracturing Pigweed's configuration. + +A different file format could be chosen over JSON. Since JSON is parsed into +only Python lists, dicts, and primitives, switching to another format that can +be parsed into the same internal structure should be trivial. + +-------------- +Open questions +-------------- +None? diff --git a/seed/BUILD.gn b/seed/BUILD.gn index e2f12f6ad..6a5158fc6 100644 --- a/seed/BUILD.gn +++ b/seed/BUILD.gn @@ -21,6 +21,7 @@ pw_doc_group("docs") { group_deps = [ ":0001", ":0002", + ":0101", ":0102", ] } @@ -34,6 +35,10 @@ pw_doc_group("0002") { sources = [ "0002-template.rst" ] } +pw_doc_group("0101") { + sources = [ "0101-pigweed.json.rst" ] +} + pw_doc_group("0102") { sources = [ "0102-module-docs.rst" ] } diff --git a/targets/default_config.BUILD b/targets/default_config.BUILD index 5c7fcf9e9..b2d33d5a5 100644 --- a/targets/default_config.BUILD +++ b/targets/default_config.BUILD @@ -170,3 +170,8 @@ label_flag( name = "pw_trace_backend", build_setting_default = "@pigweed//pw_trace:backend_multiplexer", ) + +label_flag( + name = "freertos_config", + build_setting_default = "@pigweed//third_party/freertos:freertos_config", +) diff --git a/targets/host/system_rpc_server.cc b/targets/host/system_rpc_server.cc index f3d870fc9..8ff75751a 100644 --- a/targets/host/system_rpc_server.cc +++ b/targets/host/system_rpc_server.cc @@ -35,6 +35,7 @@ uint16_t socket_port = 33000; static_assert(kMaxTransmissionUnit == hdlc::MaxEncodedFrameSize(rpc::cfg::kEncodingBufferSizeBytes)); +stream::ServerSocket server_socket; stream::SocketStream socket_stream; hdlc::FixedMtuChannelOutput<kMaxTransmissionUnit> hdlc_channel_output( @@ -58,7 +59,10 @@ void Init() { }); PW_LOG_INFO("Starting pw_rpc server on port %d", socket_port); - PW_CHECK_OK(socket_stream.Serve(socket_port)); + PW_CHECK_OK(server_socket.Listen(socket_port)); + auto accept_result = server_socket.Accept(); + PW_CHECK_OK(accept_result.status()); + socket_stream = *std::move(accept_result); } rpc::Server& Server() { return server; } diff --git a/targets/stm32f429i_disc1/py/setup.cfg b/targets/stm32f429i_disc1/py/setup.cfg index 26ec850a4..a3f377296 100644 --- a/targets/stm32f429i_disc1/py/setup.cfg +++ b/targets/stm32f429i_disc1/py/setup.cfg @@ -23,6 +23,7 @@ packages = find: zip_safe = False install_requires = pyserial>=3.5,<4.0 + types-pyserial>=3.5,<4.0 coloredlogs [options.entry_points] diff --git a/targets/stm32f429i_disc1/py/stm32f429i_disc1_utils/stm32f429i_detector.py b/targets/stm32f429i_disc1/py/stm32f429i_disc1_utils/stm32f429i_detector.py index 857408287..bc09d282b 100644 --- a/targets/stm32f429i_disc1/py/stm32f429i_disc1_utils/stm32f429i_detector.py +++ b/targets/stm32f429i_disc1/py/stm32f429i_disc1_utils/stm32f429i_detector.py @@ -16,9 +16,10 @@ import logging import typing +from typing import Optional import coloredlogs # type: ignore -import serial.tools.list_ports # type: ignore +import serial.tools.list_ports # Vendor/device ID to search for in USB devices. _ST_VENDOR_ID = 0x0483 @@ -31,7 +32,7 @@ class BoardInfo(typing.NamedTuple): """Information about a connected dev board.""" dev_name: str - serial_number: str + serial_number: Optional[str] def detect_boards() -> list: @@ -67,7 +68,7 @@ def main(): for idx, board in enumerate(boards): _LOG.info('Board %d:', idx) _LOG.info(' - Port: %s', board.dev_name) - _LOG.info(' - Serial #: %s', board.serial_number) + _LOG.info(' - Serial #: %s', board.serial_number or '<not set>') if __name__ == '__main__': diff --git a/targets/stm32f429i_disc1/py/stm32f429i_disc1_utils/unit_test_runner.py b/targets/stm32f429i_disc1/py/stm32f429i_disc1_utils/unit_test_runner.py index d79e47038..b17d62790 100755 --- a/targets/stm32f429i_disc1/py/stm32f429i_disc1_utils/unit_test_runner.py +++ b/targets/stm32f429i_disc1/py/stm32f429i_disc1_utils/unit_test_runner.py @@ -23,7 +23,7 @@ import threading from typing import List import coloredlogs # type: ignore -import serial # type: ignore +import serial from stm32f429i_disc1_utils import stm32f429i_detector # Path used to access non-python resources in this python module. diff --git a/targets/stm32f429i_disc1_stm32cube/BUILD.bazel b/targets/stm32f429i_disc1_stm32cube/BUILD.bazel index 7b64e662f..7922f2968 100644 --- a/targets/stm32f429i_disc1_stm32cube/BUILD.bazel +++ b/targets/stm32f429i_disc1_stm32cube/BUILD.bazel @@ -23,26 +23,59 @@ package(default_visibility = ["//visibility:public"]) licenses(["notice"]) pw_cc_library( + name = "freertos_config", + hdrs = [ + "config/FreeRTOSConfig.h", + ], + includes = ["config/"], + target_compatible_with = [":freertos_config_cv"], + deps = ["//third_party/freertos:config_assert"], +) + +# Constraint value corresponding to :freertos_config. +# +# If you include this in your platform definition, you will tell Bazel to use +# the :freertos_config defined above when compiling FreeRTOS. (See +# //third_party/freertos/BUILD.bazel.) If you include it in a target's +# `target_compatible_with`, you will tell Bazel the target can only be built +# for platforms that specify this FreeRTOS config. +constraint_value( + name = "freertos_config_cv", + constraint_setting = "//third_party/freertos:freertos_config_setting", +) + +# TODO(b/261506064): Additional constraint values for configuring stm32cube +# need to be added here, once constraint settings for stm32cube are defined. +platform( + name = "platform", + constraint_values = [ + ":freertos_config_cv", + "//pw_build/constraints/rtos:freertos", + "@freertos//:port_ARM_CM4F", + ], + parents = ["@bazel_embedded//platforms:cortex_m4_fpu"], +) + +pw_cc_library( name = "pre_init", srcs = [ "boot.cc", "vector_table.c", ], hdrs = [ - "config/FreeRTOSConfig.h", "config/stm32f4xx_hal_conf.h", ], - # TODO(b/261506064): Get this to build. Requires FreeRTOS. - tags = ["manual"], + target_compatible_with = [":freertos_config_cv"], deps = [ + ":freertos_config", "//pw_boot", "//pw_boot_cortex_m", "//pw_malloc", "//pw_preprocessor", "//pw_string", "//pw_sys_io_stm32cube", - "//third_party/freertos", "//third_party/stm32cube", + "@freertos", ], ) @@ -51,12 +84,11 @@ pw_cc_binary( srcs = [ "main.cc", ], - # TODO(b/261506064): Get this to build. Requires FreeRTOS. - tags = ["manual"], + target_compatible_with = [":freertos_config_cv"], deps = [ "//pw_thread:thread", "//pw_thread:thread_core", "//pw_thread_freertos:thread", - "//third_party/freertos", + "@freertos", ], ) diff --git a/third_party/emboss/BUILD.gn b/third_party/emboss/BUILD.gn index 89df5c423..a0ef87dda 100644 --- a/third_party/emboss/BUILD.gn +++ b/third_party/emboss/BUILD.gn @@ -18,7 +18,20 @@ import("$dir_pw_build/python.gni") import("$dir_pw_docgen/docs.gni") import("$dir_pw_third_party/emboss/emboss.gni") +config("default_config") { + # EMBOSS_DCHECK is used as an assert() for logic embedded in Emboss, where + # EMBOSS_CHECK is used to check preconditions on application logic (e.g. + # Write() checks the [requires: ...] attribute). + defines = [ + "EMBOSS_CHECK=PW_DCHECK", + "EMBOSS_CHECK_ABORTS", + "EMBOSS_DCHECK=PW_DCHECK", + "EMBOSS_DCHECK_ABORTS", + ] +} + source_set("cpp_utils") { + public_configs = [ pw_third_party_emboss_CONFIG ] sources = [ "$dir_pw_third_party_emboss/runtime/cpp/emboss_arithmetic.h", "$dir_pw_third_party_emboss/runtime/cpp/emboss_arithmetic_all_known_generated.h", diff --git a/third_party/emboss/docs.rst b/third_party/emboss/docs.rst index 59ab75408..1cabc8aae 100644 --- a/third_party/emboss/docs.rst +++ b/third_party/emboss/docs.rst @@ -18,7 +18,7 @@ Git submodule: .. code-block:: sh - git submodule add https://github.com/google/emboss.git third_party/emboss/src + git submodule add https://github.com/google/emboss.git third_party/emboss/src Next, set the GN variable ``dir_pw_third_party_emboss`` to the path of your Emboss installation. If using the submodule path from above, add the following to the @@ -26,7 +26,19 @@ installation. If using the submodule path from above, add the following to the .. code-block:: - dir_pw_third_party_emboss = "//third_party/emboss/src" + dir_pw_third_party_emboss = "//third_party/emboss/src" + +.. + inclusive-language: disable + +Optionally, configure the Emboss defines documented at +`dir_pw_third_party_emboss/runtime/cpp/emboss_defines.h +<https://github.com/google/emboss/blob/master/runtime/cpp/emboss_defines.h>`_ +by setting the ``pw_third_party_emboss_CONFIG`` variable to a config that +overrides the defines. By default, checks will use PW_DCHECK. + +.. + inclusive-language: enable ------------ Using Emboss diff --git a/third_party/emboss/emboss.gni b/third_party/emboss/emboss.gni index 4f9062792..93d8396d7 100644 --- a/third_party/emboss/emboss.gni +++ b/third_party/emboss/emboss.gni @@ -12,8 +12,14 @@ # License for the specific language governing permissions and limitations under # the License. +import("//build_overrides/pigweed.gni") + declare_args() { # If compiling with Emboss, this variable is set to the path to the Emboss # source code. dir_pw_third_party_emboss = "" + + # config target for overriding Emboss defines (e.g. EMBOSS_CHECK). + pw_third_party_emboss_CONFIG = + "$dir_pigweed/third_party/emboss:default_config" } diff --git a/third_party/freertos/BUILD.bazel b/third_party/freertos/BUILD.bazel index f635a0f6d..01256d41b 100644 --- a/third_party/freertos/BUILD.bazel +++ b/third_party/freertos/BUILD.bazel @@ -13,7 +13,7 @@ # the License. load( - "//pw_build:pigweed.bzl", + "@pigweed//pw_build:pigweed.bzl", "pw_cc_library", ) @@ -28,6 +28,154 @@ pw_cc_library( ], includes = ["public"], deps = [ - "//pw_assert", + "@pigweed//pw_assert", ], ) + +constraint_setting( + name = "port", +) + +constraint_value( + name = "port_ARM_CM7", + constraint_setting = ":port", +) + +constraint_value( + name = "port_ARM_CM4F", + constraint_setting = ":port", +) + +pw_cc_library( + name = "freertos", + srcs = [ + "croutine.c", + "event_groups.c", + "list.c", + "queue.c", + "stream_buffer.c", + "timers.c", + ] + select({ + ":port_ARM_CM4F": ["portable/GCC/ARM_CM4F/port.c"], + ":port_ARM_CM7": ["portable/GCC/ARM_CM7/r0p1/port.c"], + "//conditions:default": [], + }), + includes = ["include/"] + select({ + ":port_ARM_CM4F": ["portable/GCC/ARM_CM4F"], + ":port_ARM_CM7": ["portable/GCC/ARM_CM7/r0p1"], + "//conditions:default": [], + }), + textual_hdrs = [ + "include/FreeRTOS.h", + "include/StackMacros.h", + "include/croutine.h", + "include/deprecated_definitions.h", + "include/event_groups.h", + "include/list.h", + "include/message_buffer.h", + "include/mpu_wrappers.h", + "include/portable.h", + "include/projdefs.h", + "include/queue.h", + "include/semphr.h", + "include/stack_macros.h", + "include/stream_buffer.h", + "include/task.h", + "include/timers.h", + ] + select({ + ":port_ARM_CM4F": ["portable/GCC/ARM_CM4F/portmacro.h"], + ":port_ARM_CM7": ["portable/GCC/ARM_CM7/r0p1/portmacro.h"], + "//conditions:default": [], + }), + deps = [ + ":pigweed_tasks_c", + "@pigweed_config//:freertos_config", + ], + # Required because breaking out tasks_c results in the dependencies between + # the libraries not being quite correct: to link pigweed_tasks_c you + # actually need a bunch of the source files from here (e.g., list.c). + alwayslink = 1, +) + +# Constraint setting used to determine if task statics should be disabled. +constraint_setting( + name = "disable_tasks_statics_setting", + default_constraint_value = ":no_disable_task_statics", +) + +constraint_value( + name = "disable_task_statics", + constraint_setting = ":disable_tasks_statics_setting", +) + +constraint_value( + name = "no_disable_task_statics", + constraint_setting = ":disable_tasks_statics_setting", +) + +pw_cc_library( + name = "pigweed_tasks_c", + srcs = ["tasks.c"], + defines = select({ + ":disable_task_statics": [ + "PW_THIRD_PARTY_FREERTOS_NO_STATICS=1", + ], + "//conditions:default": [], + }), + includes = [ + "include/", + ] + select({ + ":port_ARM_CM4F": ["portable/GCC/ARM_CM4F/"], + ":port_ARM_CM7": ["portable/GCC/ARM_CM7/r0p1/"], + "//conditions:default": [], + }), + local_defines = select({ + ":disable_task_statics": [ + "static=", + ], + "//conditions:default": [], + }), + # tasks.c transitively includes all these headers :/ + textual_hdrs = [ + "include/FreeRTOS.h", + "include/portable.h", + "include/projdefs.h", + "include/list.h", + "include/deprecated_definitions.h", + "include/mpu_wrappers.h", + "include/stack_macros.h", + "include/task.h", + "include/timers.h", + ] + select({ + ":port_ARM_CM4F": ["portable/GCC/ARM_CM4F/portmacro.h"], + ":port_ARM_CM7": ["portable/GCC/ARM_CM7/r0p1/portmacro.h"], + "//conditions:default": [], + }), + deps = ["@pigweed_config//:freertos_config"], +) + +# Constraint setting used to select the FreeRTOSConfig version. +constraint_setting( + name = "freertos_config_setting", +) + +alias( + name = "freertos_config", + actual = select({ + "@pigweed//targets/stm32f429i_disc1_stm32cube:freertos_config_cv": "@pigweed//targets/stm32f429i_disc1_stm32cube:freertos_config", + "//conditions:default": "default_freertos_config", + }), +) + +pw_cc_library( + name = "default_freertos_config", + # The "default" config is not compatible with any configuration: you can't + # build FreeRTOS without choosing a config. + target_compatible_with = ["@platforms//:incompatible"], +) + +# Exported for +# pw_thread_freertos/py/pw_thread_freertos/generate_freertos_tsktcb.py +exports_files( + ["tasks.c"], +) diff --git a/third_party/freertos/docs.rst b/third_party/freertos/docs.rst index b43e71ec2..695dd7d32 100644 --- a/third_party/freertos/docs.rst +++ b/third_party/freertos/docs.rst @@ -10,8 +10,8 @@ FreeRTOS, including Pigweed backend modules which depend on FreeRTOS. ------------- Build Support ------------- -This module provides support to compile FreeRTOS with GN and CMake. This is -required when compiling backends modules for FreeRTOS. +This module provides support to compile FreeRTOS with GN, CMake, and Bazel. +This is required when compiling backends modules for FreeRTOS. GN == @@ -39,6 +39,25 @@ In order to use this you are expected to set the following variables from #. Set ``pw_third_party_freertos_PORT`` to a library target which provides the FreeRTOS port specific includes and sources. +Bazel +===== +In Bazel, the FreeRTOS build is configured through `constraint_settings +<https://bazel.build/reference/be/platform#constraint_setting>`_. The `platform +<https://bazel.build/extending/platforms>`_ you are building for must specify +values for the following settings: + +* ``//third_party/freertos:port``, to set which FreeRTOS port to use. You can + select a value from those defined in ``third_party/freertos/BUILD.bazel``. +* ``//third_party/freertos:disable_task_statics_setting``, to determine + whether statics should be disabled during compilation of the tasks.c source + file (see next section). This setting has only two possible values, also + defined in ``third_party/freertos/BUILD.bazel``. + +In addition, you need to set the ``@pigweed_config//:freertos_config`` label +flag to point to the library target providing the FreeRTOS config header. See +:ref:`docs-build_system-bazel_configuration` for a discussion of how to work +with ``@pigweed_config``. + .. _third_party-freertos_disable_task_statics: @@ -48,10 +67,12 @@ In order to link against internal kernel data structures through the use of extern "C", statics can be optionally disabled for the tasks.c source file to enable use of things like pw_thread_freertos/util.h's ``ForEachThread``. -To facilitate this, Pigweed offers an opt-in option which can be enabled by -configuring GN through -``pw_third_party_freertos_DISABLE_TASKS_STATICS = true`` or CMake through -``set(pw_third_party_freertos_DISABLE_TASKS_STATICS ON CACHE BOOL "" FORCE)``. +To facilitate this, Pigweed offers an opt-in option which can be enabled, + +* in GN through ``pw_third_party_freertos_DISABLE_TASKS_STATICS = true``, +* in CMake through ``set(pw_third_party_freertos_DISABLE_TASKS_STATICS ON CACHE BOOL "" FORCE)``, +* in Bazel through ``//third_party/freertos:disable_task_statics``. + This redefines ``static`` to nothing for the ``Source/tasks.c`` FreeRTOS source file when building through ``$dir_pw_third_party/freertos`` in GN and through ``pw_third_party.freertos`` in CMake. diff --git a/third_party/micro_ecc/BUILD.gn b/third_party/micro_ecc/BUILD.gn index 4ead9fd64..39cfd09ea 100644 --- a/third_party/micro_ecc/BUILD.gn +++ b/third_party/micro_ecc/BUILD.gn @@ -29,8 +29,37 @@ if (dir_pw_third_party_micro_ecc != "") { defines = [ "uECC_SUPPORT_COMPRESSED_POINT=0" ] } + # Endianess is a public configuration for uECC as it determines how large + # integers are interpreted in uECC public APIs. + # + # Big endian is a lot more common and thus is recommended unless you are + # really resource-constrained or another uECC client expects little + # endian. + config("big_endian_config") { + defines = [ "uECC_VLI_NATIVE_LITTLE_ENDIAN=0" ] + } + + # Little endian can reduce call stack usage in native little endian + # execution environments (as determined by processor state, memory + # access config etc.) + config("little_endian_config") { + defines = [ "uECC_VLI_NATIVE_LITTLE_ENDIAN=1" ] + } + pw_source_set("micro_ecc") { - public_configs = [ ":public_config" ] + public_configs = [ + ":big_endian_config", + ":public_config", + ] + configs = [ ":internal_config" ] + sources = [ "$dir_pw_third_party_micro_ecc/uECC.c" ] + } + + pw_source_set("micro_ecc_little_endian") { + public_configs = [ + ":little_endian_config", + ":public_config", + ] configs = [ ":internal_config" ] sources = [ "$dir_pw_third_party_micro_ecc/uECC.c" ] } |