pw_log_tokenized: Add config header

- Make the tokenization macro and log metadata bits configurable.
- Make the pw_log_tokenized tests a regular test instad of a facade
  test by selecting an alternate tokenization macro.

Change-Id: Ie03e047769ef08d3bc08400861b158307366a9b3
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/39982
Pigweed-Auto-Submit: Wyatt Hepler <hepler@google.com>
Reviewed-by: Keir Mierle <keir@google.com>
Commit-Queue: Auto-Submit <auto-submit@pigweed.google.com.iam.gserviceaccount.com>

7 files changed, 154 insertions, 86 deletions

diff --git a/pw_log_tokenized/BUILD b/pw_log_tokenized/BUILD
index eac9f2915..41b0747ba 100644
--- a/pw_log_tokenized/BUILD
+++ b/pw_log_tokenized/BUILD
@@ -25,6 +25,7 @@ licenses(["notice"])  # Apache License 2.0
 pw_cc_library(
     name = "headers",
     hdrs = [
+        "public/pw_log_tokenized/config.h",
         "public/pw_log_tokenized/log_tokenized.h",
         "public_overrides/pw_log_backend/log_backend.h",
     ],
diff --git a/pw_log_tokenized/BUILD.gn b/pw_log_tokenized/BUILD.gn
index 1feee292e..d51a510ad 100644
--- a/pw_log_tokenized/BUILD.gn
+++ b/pw_log_tokenized/BUILD.gn
@@ -14,13 +14,20 @@
 
 import("//build_overrides/pigweed.gni")
 
+import("$dir_pw_build/module_config.gni")
 import("$dir_pw_build/target_types.gni")
 import("$dir_pw_docgen/docs.gni")
 import("$dir_pw_log/backend.gni")
 import("$dir_pw_tokenizer/backend.gni")
-import("$dir_pw_unit_test/facade_test.gni")
 import("$dir_pw_unit_test/test.gni")
 
+declare_args() {
+  # The build target that overrides the default configuration options for this
+  # module. This should point to a source set that provides defines through a
+  # public config (which may -include a file or add defines directly).
+  pw_log_tokenized_CONFIG = pw_build_DEFAULT_MODULE_CONFIG
+}
+
 config("public_includes") {
   include_dirs = [ "public" ]
   visibility = [ ":*" ]
@@ -49,8 +56,12 @@ pw_source_set("pw_log_tokenized") {
     "$dir_pw_log:facade",
     "$dir_pw_tokenizer:global_handler_with_payload.facade",
     dir_pw_preprocessor,
+    pw_log_tokenized_CONFIG,
+  ]
+  public = [
+    "public/pw_log_tokenized/config.h",
+    "public/pw_log_tokenized/log_tokenized.h",
   ]
-  public = [ "public/pw_log_tokenized/log_tokenized.h" ]
 }
 
 # This target provides the backend for pw_log.
@@ -79,19 +90,9 @@ pw_test_group("tests") {
   tests = [ ":log_tokenized_test" ]
 }
 
-pw_facade_test("log_tokenized_test") {
-  build_args = {
-    # The test provides pw_tokenizer_GLOBAL_HANDLER_WITH_PAYLOAD_BACKEND in the
-    # .cc file, but we can't depend on the test as it creates a circular
-    # dependency.
-    pw_tokenizer_GLOBAL_HANDLER_WITH_PAYLOAD_BACKEND = "$dir_pw_build:empty"
-  }
+pw_test("log_tokenized_test") {
   sources = [ "test.cc" ]
-  deps = [
-    ":log_backend",
-    ":pw_log_tokenized",
-  ]
-  enable_if = pw_log_BACKEND == "$dir_pw_log_tokenized:log_backend"
+  deps = [ ":pw_log_tokenized" ]
 }
 
 pw_doc_group("docs") {
diff --git a/pw_log_tokenized/docs.rst b/pw_log_tokenized/docs.rst
index e5242b81b..f64f52468 100644
--- a/pw_log_tokenized/docs.rst
+++ b/pw_log_tokenized/docs.rst
@@ -9,10 +9,12 @@ connects ``pw_log`` to ``pw_tokenizer``.
 C++ backend
 ===========
 ``pw_log_tokenized`` provides a backend for ``pw_log`` that tokenizes log
-messages with the ``pw_tokenizer`` module. Log messages are tokenized and passed
-to the ``pw_tokenizer_HandleEncodedMessageWithPayload`` function. For maximum
-efficiency, the log level, 16-bit tokenized module name, and flags bits are
-passed through the payload argument.
+messages with the ``pw_tokenizer`` module. By default, log messages are
+tokenized with the ``PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD`` macro.
+The log level, 16-bit tokenized module name, and flags bits are passed through
+the payload argument. The macro eventually passes logs to the
+``pw_tokenizer_HandleEncodedMessageWithPayload`` function, which must be
+implemented by the application.
 
 Example implementation:
 
@@ -35,7 +37,14 @@ Example implementation:
      }
    }
 
-See the documentation for ``pw_tokenizer`` for further details.
+See the documentation for :ref:`module-pw_tokenizer` for further details.
+
+Applications may select a different macro than
+``PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD`` by setting the
+``PW_LOG_TOKENIZED_ENCODE_MESSAGE`` config macro. This macro should take
+arguments equivalent to ``PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD``:
+
+  .. c:function:: PW_LOG_TOKENIZED_ENCODE_MESSAGE(pw_tokenizer_Payload log_metadata, const char* message, ...)
 
 Build targets
 -------------
diff --git a/pw_log_tokenized/public/pw_log_tokenized/config.h b/pw_log_tokenized/public/pw_log_tokenized/config.h
new file mode 100644
index 000000000..9c92d9f64
--- /dev/null
+++ b/pw_log_tokenized/public/pw_log_tokenized/config.h
@@ -0,0 +1,58 @@
+// Copyright 2021 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include <assert.h>
+
+#include "pw_log/options.h"
+#include "pw_preprocessor/concat.h"
+
+// This macro takes the PW_LOG format string and optionally transforms it. By
+// default, the PW_LOG_MODULE_NAME is prepended to the string if present.
+#ifndef PW_LOG_TOKENIZED_FORMAT_STRING
+
+#define PW_LOG_TOKENIZED_FORMAT_STRING(string) \
+  PW_CONCAT(PW_LOG_TOKENIZED_FMT_, PW_LOG_MODULE_NAME_DEFINED)(string)
+
+#define PW_LOG_TOKENIZED_FMT_0(string) string
+#define PW_LOG_TOKENIZED_FMT_1(string) PW_LOG_MODULE_NAME " " string
+
+#endif  // PW_LOG_TOKENIZED_FORMAT_STRING
+
+// The log level, module token, and flag bits are packed into the tokenizer's
+// payload argument, which is typically 32 bits. These macros specify the number
+// of bits to use for each field.
+#ifndef PW_LOG_TOKENIZED_LEVEL_BITS
+#define PW_LOG_TOKENIZED_LEVEL_BITS 6
+#endif  // PW_LOG_TOKENIZED_LEVEL_BITS
+
+#ifndef PW_LOG_TOKENIZED_MODULE_BITS
+#define PW_LOG_TOKENIZED_MODULE_BITS 16
+#endif  // PW_LOG_TOKENIZED_MODULE_BITS
+
+#ifndef PW_LOG_TOKENIZED_FLAG_BITS
+#define PW_LOG_TOKENIZED_FLAG_BITS 10
+#endif  // PW_LOG_TOKENIZED_FLAG_BITS
+
+static_assert((PW_LOG_TOKENIZED_LEVEL_BITS + PW_LOG_TOKENIZED_MODULE_BITS +
+               PW_LOG_TOKENIZED_FLAG_BITS) == 32,
+              "Log metadata must fit in a 32-bit integer");
+
+// The macro to use to tokenize the log and its arguments. Defaults to
+// PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD. Projects may define their own
+// version of this macro that uses a different underlying function, if desired.
+#ifndef PW_LOG_TOKENIZED_ENCODE_MESSAGE
+#define PW_LOG_TOKENIZED_ENCODE_MESSAGE \
+  PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD
+#endif  // PW_LOG_TOKENIZED_ENCODE_MESSAGE
diff --git a/pw_log_tokenized/public/pw_log_tokenized/log_tokenized.h b/pw_log_tokenized/public/pw_log_tokenized/log_tokenized.h
index 02babfa9f..34acb74b6 100644
--- a/pw_log_tokenized/public/pw_log_tokenized/log_tokenized.h
+++ b/pw_log_tokenized/public/pw_log_tokenized/log_tokenized.h
@@ -1,4 +1,4 @@
-// Copyright 2020 The Pigweed Authors
+// Copyright 2021 The Pigweed Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License"); you may not
 // use this file except in compliance with the License. You may obtain a copy of
@@ -13,15 +13,14 @@
 // the License.
 #pragma once
 
-#include <assert.h>
 #include <stdint.h>
 
-#include "pw_log/options.h"
-#include "pw_preprocessor/concat.h"
+#include "pw_log_tokenized/config.h"
 #include "pw_tokenizer/tokenize_to_global_handler_with_payload.h"
 
 // This macro implements PW_LOG using
-// PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD. The log level, module token, and
+// PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD or an equivalent alternate macro
+// provided by PW_LOG_TOKENIZED_ENCODE_MESSAGE. The log level, module token, and
 // flags are packed into the payload argument.
 //
 // Two strings are tokenized in this macro:
@@ -43,46 +42,24 @@
 //     }
 //   }
 //
-#define PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(                       \
-    level, flags, message, ...)                                                \
-  do {                                                                         \
-    _PW_TOKENIZER_CONST uintptr_t _pw_log_module_token =                       \
-        PW_TOKENIZE_STRING_MASK("pw_log_module_names",                         \
-                                ((1u << _PW_LOG_TOKENIZED_MODULE_BITS) - 1u),  \
-                                PW_LOG_MODULE_NAME);                           \
-    PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD(                                \
-        ((uintptr_t)(level) |                                                  \
-         (_pw_log_module_token << _PW_LOG_TOKENIZED_LEVEL_BITS) |              \
-         ((uintptr_t)(flags)                                                   \
-          << (_PW_LOG_TOKENIZED_LEVEL_BITS + _PW_LOG_TOKENIZED_MODULE_BITS))), \
-        PW_LOG_TOKENIZED_FORMAT_STRING(message),                               \
-        __VA_ARGS__);                                                          \
+#define PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(                     \
+    level, flags, message, ...)                                              \
+  do {                                                                       \
+    _PW_TOKENIZER_CONST uintptr_t _pw_log_module_token =                     \
+        PW_TOKENIZE_STRING_MASK("pw_log_module_names",                       \
+                                ((1u << PW_LOG_TOKENIZED_MODULE_BITS) - 1u), \
+                                PW_LOG_MODULE_NAME);                         \
+    PW_LOG_TOKENIZED_ENCODE_MESSAGE(                                         \
+        ((uintptr_t)(level) |                                                \
+         (_pw_log_module_token << PW_LOG_TOKENIZED_LEVEL_BITS) |             \
+         ((uintptr_t)(flags)                                                 \
+          << (PW_LOG_TOKENIZED_LEVEL_BITS + PW_LOG_TOKENIZED_MODULE_BITS))), \
+        PW_LOG_TOKENIZED_FORMAT_STRING(message),                             \
+        __VA_ARGS__);                                                        \
   } while (0)
 
-// By default, log format strings include the PW_LOG_MODULE_NAME, if defined.
-#ifndef PW_LOG_TOKENIZED_FORMAT_STRING
-
-#define PW_LOG_TOKENIZED_FORMAT_STRING(string) \
-  PW_CONCAT(_PW_LOG_TOKENIZED_FMT_, PW_LOG_MODULE_NAME_DEFINED)(string)
-
-#define _PW_LOG_TOKENIZED_FMT_0(string) string
-#define _PW_LOG_TOKENIZED_FMT_1(string) PW_LOG_MODULE_NAME " " string
-
-#endif  // PW_LOG_TOKENIZED_FORMAT_STRING
-
-// The log level, module token, and flag bits are packed into the tokenizer's
-// payload argument, which is typically 32 bits. These macros specify the number
-// of bits to use for each field.
-#define _PW_LOG_TOKENIZED_LEVEL_BITS 6
-#define _PW_LOG_TOKENIZED_MODULE_BITS 16
-#define _PW_LOG_TOKENIZED_FLAG_BITS 10
-
 #ifdef __cplusplus
 
-static_assert((_PW_LOG_TOKENIZED_LEVEL_BITS + _PW_LOG_TOKENIZED_MODULE_BITS +
-               _PW_LOG_TOKENIZED_FLAG_BITS) == 32,
-              "Log metadata must fit in a 32-bit integer");
-
 namespace pw {
 namespace log_tokenized {
 namespace internal {
@@ -133,9 +110,9 @@ class GenericMetadata {
 
 }  // namespace internal
 
-using Metadata = internal::GenericMetadata<_PW_LOG_TOKENIZED_LEVEL_BITS,
-                                           _PW_LOG_TOKENIZED_MODULE_BITS,
-                                           _PW_LOG_TOKENIZED_FLAG_BITS>;
+using Metadata = internal::GenericMetadata<PW_LOG_TOKENIZED_LEVEL_BITS,
+                                           PW_LOG_TOKENIZED_MODULE_BITS,
+                                           PW_LOG_TOKENIZED_FLAG_BITS>;
 
 }  // namespace log_tokenized
 }  // namespace pw
diff --git a/pw_log_tokenized/public_overrides/pw_log_backend/log_backend.h b/pw_log_tokenized/public_overrides/pw_log_backend/log_backend.h
index 02361b2c8..7b1cac109 100644
--- a/pw_log_tokenized/public_overrides/pw_log_backend/log_backend.h
+++ b/pw_log_tokenized/public_overrides/pw_log_backend/log_backend.h
@@ -1,4 +1,4 @@
-// Copyright 2020 The Pigweed Authors
+// Copyright 2021 The Pigweed Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License"); you may not
 // use this file except in compliance with the License. You may obtain a copy of
@@ -16,9 +16,10 @@
 // the PW_LOG macro as the tokenized logging macro.
 #pragma once
 
+#include "pw_log_tokenized/config.h"
 #include "pw_log_tokenized/log_tokenized.h"
 
 #define PW_HANDLE_LOG PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD
 
-#define PW_LOG_LEVEL_BITS _PW_LOG_TOKENIZED_LEVEL_BITS
-#define PW_LOG_FLAG_BITS _PW_LOG_TOKENIZED_FLAG_BITS
+#define PW_LOG_LEVEL_BITS PW_LOG_TOKENIZED_LEVEL_BITS
+#define PW_LOG_FLAG_BITS PW_LOG_TOKENIZED_FLAG_BITS
diff --git a/pw_log_tokenized/test.cc b/pw_log_tokenized/test.cc
index 6b5f976a8..7a843f47b 100644
--- a/pw_log_tokenized/test.cc
+++ b/pw_log_tokenized/test.cc
@@ -1,4 +1,4 @@
-// Copyright 2020 The Pigweed Authors
+// Copyright 2021 The Pigweed Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License"); you may not
 // use this file except in compliance with the License. You may obtain a copy of
@@ -14,27 +14,48 @@
 
 #define PW_LOG_MODULE_NAME "This is the log module name!"
 
+// Create a fake version of the tokenization macro.
+#undef PW_LOG_TOKENIZED_ENCODE_MESSAGE
+#define PW_LOG_TOKENIZED_ENCODE_MESSAGE(payload, message, ...) \
+  CaptureTokenizerArgs(payload,                                \
+                       PW_MACRO_ARG_COUNT(__VA_ARGS__),        \
+                       message PW_COMMA_ARGS(__VA_ARGS__))
+
 #include <cstring>
 #include <string_view>
+#include <tuple>
 
 #include "gtest/gtest.h"
 #include "pw_log_tokenized/log_tokenized.h"
+#include "pw_preprocessor/arguments.h"
+#include "pw_preprocessor/compiler.h"
 
 namespace pw::log_tokenized {
 namespace {
 
-Metadata metadata(0);
-size_t encoded_data_size = 0;
+struct {
+  Metadata metadata = Metadata(0);
+  const char* format_string = "";
+  size_t arg_count = 0;
+} last_log{};
+
+void CaptureTokenizerArgs(pw_tokenizer_Payload payload,
+                          size_t arg_count,
+                          const char* message,
+                          ...) PW_PRINTF_FORMAT(3, 4);
 
-extern "C" void pw_tokenizer_HandleEncodedMessageWithPayload(
-    pw_tokenizer_Payload payload, const uint8_t[], size_t size) {
-  metadata = payload;
-  encoded_data_size = size;
+void CaptureTokenizerArgs(pw_tokenizer_Payload payload,
+                          size_t arg_count,
+                          const char* message,
+                          ...) {
+  last_log.metadata = payload;
+  last_log.format_string = message;
+  last_log.arg_count = arg_count;
 }
 
 constexpr uintptr_t kModuleToken =
     PW_TOKENIZER_STRING_TOKEN(PW_LOG_MODULE_NAME) &
-    ((1u << _PW_LOG_TOKENIZED_MODULE_BITS) - 1);
+    ((1u << PW_LOG_TOKENIZED_MODULE_BITS) - 1);
 
 constexpr Metadata test1 = Metadata::Set<0, 0, 0>();
 static_assert(test1.level() == 0);
@@ -53,26 +74,26 @@ static_assert(test3.flags() == 1023);
 
 TEST(LogTokenized, LogMetadata_Zero) {
   PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(0, 0, "hello");
-  EXPECT_EQ(metadata.level(), 0u);
-  EXPECT_EQ(metadata.flags(), 0u);
-  EXPECT_EQ(metadata.module(), kModuleToken);
-  EXPECT_EQ(encoded_data_size, 4u /* token */);
+  EXPECT_EQ(last_log.metadata.level(), 0u);
+  EXPECT_EQ(last_log.metadata.flags(), 0u);
+  EXPECT_EQ(last_log.metadata.module(), kModuleToken);
+  EXPECT_EQ(last_log.arg_count, 0u);
 }
 
 TEST(LogTokenized, LogMetadata_DifferentValues) {
   PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(55, 36, "hello%s", "?");
-  EXPECT_EQ(metadata.level(), 55u);
-  EXPECT_EQ(metadata.flags(), 36u);
-  EXPECT_EQ(metadata.module(), kModuleToken);
-  EXPECT_EQ(encoded_data_size, 4u /* token */ + 2u /* encoded string */);
+  EXPECT_EQ(last_log.metadata.level(), 55u);
+  EXPECT_EQ(last_log.metadata.flags(), 36u);
+  EXPECT_EQ(last_log.metadata.module(), kModuleToken);
+  EXPECT_EQ(last_log.arg_count, 1u);
 }
 
 TEST(LogTokenized, LogMetadata_MaxValues) {
   PW_LOG_TOKENIZED_TO_GLOBAL_HANDLER_WITH_PAYLOAD(63, 1023, "hello %d", 1);
-  EXPECT_EQ(metadata.level(), 63u);
-  EXPECT_EQ(metadata.flags(), 1023u);
-  EXPECT_EQ(metadata.module(), kModuleToken);
-  EXPECT_EQ(encoded_data_size, 4u /* token */ + 1u /* encoded integer */);
+  EXPECT_EQ(last_log.metadata.level(), 63u);
+  EXPECT_EQ(last_log.metadata.flags(), 1023u);
+  EXPECT_EQ(last_log.metadata.module(), kModuleToken);
+  EXPECT_EQ(last_log.arg_count, 1u);
 }
 
 }  // namespace