aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPrimiano Tucci <primiano@google.com>2021-06-10 12:15:42 +0000
committerAutomerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>2021-06-10 12:15:42 +0000
commitfaba89b6c475a3bc5e3e1591610b47881e573681 (patch)
tree8482ff828baf9ffa89a544a6b6057523a3b4e75f
parente1119d64130742b8126c39ba2a955179330a9527 (diff)
parent633cf4512698f868f03d3736c30f7c4c70eb1bd0 (diff)
downloadperfetto-faba89b6c475a3bc5e3e1591610b47881e573681.tar.gz
Cherry-pick proto filter changes on sc-dev am: 633cf45126
Original change: https://googleplex-android-review.googlesource.com/c/platform/external/perfetto/+/14844417 Change-Id: I5d6b1d719097fb9f294339fa49fe7a43a884e6ed
-rw-r--r--Android.bp211
-rw-r--r--BUILD89
-rw-r--r--gn/BUILD.gn2
-rw-r--r--gn/perfetto_benchmarks.gni1
-rw-r--r--gn/perfetto_fuzzers.gni1
-rw-r--r--include/perfetto/ext/tracing/core/slice.h8
-rw-r--r--include/perfetto/protozero/proto_utils.h20
-rw-r--r--protos/perfetto/common/trace_stats.proto9
-rw-r--r--protos/perfetto/config/BUILD.gn5
-rw-r--r--protos/perfetto/config/perfetto_config.proto11
-rw-r--r--protos/perfetto/config/proto_filter.proto47
-rw-r--r--protos/perfetto/config/trace_config.proto11
-rw-r--r--protos/perfetto/trace/perfetto_trace.proto20
-rw-r--r--src/android_stats/perfetto_atoms.h1
-rw-r--r--src/perfetto_cmd/pbtxt_to_pb.cc111
-rw-r--r--src/perfetto_cmd/pbtxt_to_pb_unittest.cc2
-rw-r--r--src/protozero/filtering/BUILD.gn71
-rw-r--r--src/protozero/filtering/filter_bytecode_common.h42
-rw-r--r--src/protozero/filtering/filter_bytecode_generator.cc33
-rw-r--r--src/protozero/filtering/filter_bytecode_generator.h4
-rw-r--r--src/protozero/filtering/filter_bytecode_generator_unittest.cc2
-rw-r--r--src/protozero/filtering/filter_bytecode_parser.cc75
-rw-r--r--src/protozero/filtering/filter_bytecode_parser.h14
-rw-r--r--src/protozero/filtering/filter_bytecode_parser_fuzzer.cc34
-rw-r--r--src/protozero/filtering/filter_bytecode_parser_unittest.cc116
-rw-r--r--src/protozero/filtering/filter_util.cc323
-rw-r--r--src/protozero/filtering/filter_util.h107
-rw-r--r--src/protozero/filtering/filter_util_unittest.cc182
-rw-r--r--src/protozero/filtering/message_filter.cc310
-rw-r--r--src/protozero/filtering/message_filter.h208
-rw-r--r--src/protozero/filtering/message_filter_benchmark.cc48
-rw-r--r--src/protozero/filtering/message_filter_fuzzer.cc104
-rw-r--r--src/protozero/filtering/message_filter_unittest.cc810
-rw-r--r--src/tracing/core/BUILD.gn3
-rw-r--r--src/tracing/core/tracing_service_impl.cc83
-rw-r--r--src/tracing/core/tracing_service_impl.h11
-rw-r--r--test/configs/BUILD.gn5
-rw-r--r--test/configs/ftrace_with_filter.cfg36
-rw-r--r--tools/BUILD.gn1
-rwxr-xr-xtools/install-build-deps4
-rw-r--r--tools/proto_filter/BUILD.gn29
-rw-r--r--tools/proto_filter/proto_filter.cc294
-rwxr-xr-xtools/run_android_test6
-rw-r--r--tools/test_data.txt5
44 files changed, 3072 insertions, 437 deletions
diff --git a/Android.bp b/Android.bp
index 7f8b8e072..090910dbd 100644
--- a/Android.bp
+++ b/Android.bp
@@ -47,8 +47,6 @@ cc_binary {
":perfetto_protos_perfetto_config_process_stats_zero_gen",
":perfetto_protos_perfetto_config_profiling_cpp_gen",
":perfetto_protos_perfetto_config_profiling_zero_gen",
- ":perfetto_protos_perfetto_config_proto_filter_cpp_gen",
- ":perfetto_protos_perfetto_config_proto_filter_zero_gen",
":perfetto_protos_perfetto_config_sys_stats_cpp_gen",
":perfetto_protos_perfetto_config_sys_stats_zero_gen",
":perfetto_protos_perfetto_config_track_event_cpp_gen",
@@ -125,8 +123,6 @@ cc_binary {
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_cpp_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_cpp_gen_headers",
@@ -303,8 +299,6 @@ cc_library_shared {
":perfetto_protos_perfetto_config_process_stats_zero_gen",
":perfetto_protos_perfetto_config_profiling_cpp_gen",
":perfetto_protos_perfetto_config_profiling_zero_gen",
- ":perfetto_protos_perfetto_config_proto_filter_cpp_gen",
- ":perfetto_protos_perfetto_config_proto_filter_zero_gen",
":perfetto_protos_perfetto_config_sys_stats_cpp_gen",
":perfetto_protos_perfetto_config_sys_stats_zero_gen",
":perfetto_protos_perfetto_config_track_event_cpp_gen",
@@ -388,8 +382,6 @@ cc_library_shared {
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_cpp_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_cpp_gen_headers",
@@ -490,8 +482,6 @@ cc_library_shared {
":perfetto_protos_perfetto_config_process_stats_zero_gen",
":perfetto_protos_perfetto_config_profiling_cpp_gen",
":perfetto_protos_perfetto_config_profiling_zero_gen",
- ":perfetto_protos_perfetto_config_proto_filter_cpp_gen",
- ":perfetto_protos_perfetto_config_proto_filter_zero_gen",
":perfetto_protos_perfetto_config_sys_stats_cpp_gen",
":perfetto_protos_perfetto_config_sys_stats_zero_gen",
":perfetto_protos_perfetto_config_track_event_cpp_gen",
@@ -526,6 +516,9 @@ cc_library_shared {
":perfetto_src_ipc_common",
":perfetto_src_ipc_host",
":perfetto_src_kallsyms_kallsyms",
+ ":perfetto_src_protozero_filtering_bytecode_common",
+ ":perfetto_src_protozero_filtering_bytecode_parser",
+ ":perfetto_src_protozero_filtering_message_filter",
":perfetto_src_protozero_protozero",
":perfetto_src_traced_probes_android_log_android_log",
":perfetto_src_traced_probes_common_common",
@@ -579,8 +572,6 @@ cc_library_shared {
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_cpp_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_cpp_gen_headers",
@@ -698,8 +689,6 @@ cc_library_static {
":perfetto_protos_perfetto_config_process_stats_zero_gen",
":perfetto_protos_perfetto_config_profiling_cpp_gen",
":perfetto_protos_perfetto_config_profiling_zero_gen",
- ":perfetto_protos_perfetto_config_proto_filter_cpp_gen",
- ":perfetto_protos_perfetto_config_proto_filter_zero_gen",
":perfetto_protos_perfetto_config_sys_stats_cpp_gen",
":perfetto_protos_perfetto_config_sys_stats_zero_gen",
":perfetto_protos_perfetto_config_track_event_cpp_gen",
@@ -733,6 +722,9 @@ cc_library_static {
":perfetto_src_ipc_client",
":perfetto_src_ipc_common",
":perfetto_src_ipc_host",
+ ":perfetto_src_protozero_filtering_bytecode_common",
+ ":perfetto_src_protozero_filtering_bytecode_parser",
+ ":perfetto_src_protozero_filtering_message_filter",
":perfetto_src_protozero_protozero",
":perfetto_src_tracing_client_api_without_backends",
":perfetto_src_tracing_common",
@@ -773,8 +765,6 @@ cc_library_static {
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_cpp_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_cpp_gen_headers",
@@ -821,8 +811,6 @@ cc_library_static {
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_cpp_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_cpp_gen_headers",
@@ -897,8 +885,6 @@ cc_binary {
":perfetto_protos_perfetto_config_process_stats_zero_gen",
":perfetto_protos_perfetto_config_profiling_cpp_gen",
":perfetto_protos_perfetto_config_profiling_zero_gen",
- ":perfetto_protos_perfetto_config_proto_filter_cpp_gen",
- ":perfetto_protos_perfetto_config_proto_filter_zero_gen",
":perfetto_protos_perfetto_config_sys_stats_cpp_gen",
":perfetto_protos_perfetto_config_sys_stats_zero_gen",
":perfetto_protos_perfetto_config_track_event_cpp_gen",
@@ -966,8 +952,6 @@ cc_binary {
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_cpp_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_cpp_gen_headers",
@@ -1040,8 +1024,6 @@ cc_library_static {
":perfetto_protos_perfetto_config_process_stats_zero_gen",
":perfetto_protos_perfetto_config_profiling_cpp_gen",
":perfetto_protos_perfetto_config_profiling_zero_gen",
- ":perfetto_protos_perfetto_config_proto_filter_cpp_gen",
- ":perfetto_protos_perfetto_config_proto_filter_zero_gen",
":perfetto_protos_perfetto_config_sys_stats_cpp_gen",
":perfetto_protos_perfetto_config_sys_stats_zero_gen",
":perfetto_protos_perfetto_config_track_event_cpp_gen",
@@ -1092,6 +1074,9 @@ cc_library_static {
":perfetto_src_ipc_host",
":perfetto_src_ipc_perfetto_ipc",
":perfetto_src_kallsyms_kallsyms",
+ ":perfetto_src_protozero_filtering_bytecode_common",
+ ":perfetto_src_protozero_filtering_bytecode_parser",
+ ":perfetto_src_protozero_filtering_message_filter",
":perfetto_src_protozero_protozero",
":perfetto_src_traced_probes_android_log_android_log",
":perfetto_src_traced_probes_common_common",
@@ -1152,8 +1137,6 @@ cc_library_static {
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_cpp_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_cpp_gen_headers",
@@ -1214,8 +1197,6 @@ cc_library_static {
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_cpp_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_cpp_gen_headers",
@@ -1299,8 +1280,6 @@ cc_library_static {
":perfetto_protos_perfetto_config_process_stats_zero_gen",
":perfetto_protos_perfetto_config_profiling_cpp_gen",
":perfetto_protos_perfetto_config_profiling_zero_gen",
- ":perfetto_protos_perfetto_config_proto_filter_cpp_gen",
- ":perfetto_protos_perfetto_config_proto_filter_zero_gen",
":perfetto_protos_perfetto_config_sys_stats_cpp_gen",
":perfetto_protos_perfetto_config_sys_stats_zero_gen",
":perfetto_protos_perfetto_config_track_event_cpp_gen",
@@ -1351,6 +1330,9 @@ cc_library_static {
":perfetto_src_ipc_host",
":perfetto_src_ipc_perfetto_ipc",
":perfetto_src_kallsyms_kallsyms",
+ ":perfetto_src_protozero_filtering_bytecode_common",
+ ":perfetto_src_protozero_filtering_bytecode_parser",
+ ":perfetto_src_protozero_filtering_message_filter",
":perfetto_src_protozero_protozero",
":perfetto_src_traced_probes_android_log_android_log",
":perfetto_src_traced_probes_common_common",
@@ -1397,8 +1379,6 @@ cc_library_static {
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_cpp_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_cpp_gen_headers",
@@ -1459,8 +1439,6 @@ cc_library_static {
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_cpp_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_cpp_gen_headers",
@@ -1696,8 +1674,6 @@ cc_test {
":perfetto_protos_perfetto_config_process_stats_zero_gen",
":perfetto_protos_perfetto_config_profiling_cpp_gen",
":perfetto_protos_perfetto_config_profiling_zero_gen",
- ":perfetto_protos_perfetto_config_proto_filter_cpp_gen",
- ":perfetto_protos_perfetto_config_proto_filter_zero_gen",
":perfetto_protos_perfetto_config_sys_stats_cpp_gen",
":perfetto_protos_perfetto_config_sys_stats_zero_gen",
":perfetto_protos_perfetto_config_track_event_cpp_gen",
@@ -1763,6 +1739,9 @@ cc_test {
":perfetto_src_profiling_memory_ring_buffer",
":perfetto_src_profiling_memory_scoped_spinlock",
":perfetto_src_profiling_memory_wire_protocol",
+ ":perfetto_src_protozero_filtering_bytecode_common",
+ ":perfetto_src_protozero_filtering_bytecode_parser",
+ ":perfetto_src_protozero_filtering_message_filter",
":perfetto_src_protozero_protozero",
":perfetto_src_trace_processor_analysis_analysis",
":perfetto_src_trace_processor_containers_containers",
@@ -1857,8 +1836,6 @@ cc_test {
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_cpp_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_cpp_gen_headers",
@@ -2419,7 +2396,6 @@ genrule {
"protos/perfetto/config/profiling/heapprofd_config.proto",
"protos/perfetto/config/profiling/java_hprof_config.proto",
"protos/perfetto/config/profiling/perf_event_config.proto",
- "protos/perfetto/config/proto_filter.proto",
"protos/perfetto/config/stress_test_config.proto",
"protos/perfetto/config/sys_stats/sys_stats_config.proto",
"protos/perfetto/config/test_config.proto",
@@ -3282,112 +3258,6 @@ genrule {
],
}
-// GN: //protos/perfetto/config:proto_filter_cpp
-genrule {
- name: "perfetto_protos_perfetto_config_proto_filter_cpp_gen",
- srcs: [
- "protos/perfetto/config/proto_filter.proto",
- ],
- tools: [
- "aprotoc",
- "perfetto_src_protozero_protoc_plugin_cppgen_plugin",
- ],
- cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --plugin=protoc-gen-plugin=$(location perfetto_src_protozero_protoc_plugin_cppgen_plugin) --plugin_out=wrapper_namespace=gen:$(genDir)/external/perfetto/ $(in)",
- out: [
- "external/perfetto/protos/perfetto/config/proto_filter.gen.cc",
- ],
-}
-
-// GN: //protos/perfetto/config:proto_filter_cpp
-genrule {
- name: "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- srcs: [
- "protos/perfetto/config/proto_filter.proto",
- ],
- tools: [
- "aprotoc",
- "perfetto_src_protozero_protoc_plugin_cppgen_plugin",
- ],
- cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --plugin=protoc-gen-plugin=$(location perfetto_src_protozero_protoc_plugin_cppgen_plugin) --plugin_out=wrapper_namespace=gen:$(genDir)/external/perfetto/ $(in)",
- out: [
- "external/perfetto/protos/perfetto/config/proto_filter.gen.h",
- ],
- export_include_dirs: [
- ".",
- "protos",
- ],
-}
-
-// GN: //protos/perfetto/config:proto_filter_lite
-genrule {
- name: "perfetto_protos_perfetto_config_proto_filter_lite_gen",
- srcs: [
- "protos/perfetto/config/proto_filter.proto",
- ],
- tools: [
- "aprotoc",
- ],
- cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --cpp_out=lite=true:$(genDir)/external/perfetto/ $(in)",
- out: [
- "external/perfetto/protos/perfetto/config/proto_filter.pb.cc",
- ],
-}
-
-// GN: //protos/perfetto/config:proto_filter_lite
-genrule {
- name: "perfetto_protos_perfetto_config_proto_filter_lite_gen_headers",
- srcs: [
- "protos/perfetto/config/proto_filter.proto",
- ],
- tools: [
- "aprotoc",
- ],
- cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --cpp_out=lite=true:$(genDir)/external/perfetto/ $(in)",
- out: [
- "external/perfetto/protos/perfetto/config/proto_filter.pb.h",
- ],
- export_include_dirs: [
- ".",
- "protos",
- ],
-}
-
-// GN: //protos/perfetto/config:proto_filter_zero
-genrule {
- name: "perfetto_protos_perfetto_config_proto_filter_zero_gen",
- srcs: [
- "protos/perfetto/config/proto_filter.proto",
- ],
- tools: [
- "aprotoc",
- "protozero_plugin",
- ],
- cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --plugin=protoc-gen-plugin=$(location protozero_plugin) --plugin_out=wrapper_namespace=pbzero:$(genDir)/external/perfetto/ $(in)",
- out: [
- "external/perfetto/protos/perfetto/config/proto_filter.pbzero.cc",
- ],
-}
-
-// GN: //protos/perfetto/config:proto_filter_zero
-genrule {
- name: "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
- srcs: [
- "protos/perfetto/config/proto_filter.proto",
- ],
- tools: [
- "aprotoc",
- "protozero_plugin",
- ],
- cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --plugin=protoc-gen-plugin=$(location protozero_plugin) --plugin_out=wrapper_namespace=pbzero:$(genDir)/external/perfetto/ $(in)",
- out: [
- "external/perfetto/protos/perfetto/config/proto_filter.pbzero.h",
- ],
- export_include_dirs: [
- ".",
- "protos",
- ],
-}
-
// GN: //protos/perfetto/config/sys_stats:cpp
genrule {
name: "perfetto_protos_perfetto_config_sys_stats_cpp_gen",
@@ -7414,6 +7284,11 @@ filegroup {
],
}
+// GN: //src/protozero/filtering:bytecode_common
+filegroup {
+ name: "perfetto_src_protozero_filtering_bytecode_common",
+}
+
// GN: //src/protozero/filtering:bytecode_generator
filegroup {
name: "perfetto_src_protozero_filtering_bytecode_generator",
@@ -7430,12 +7305,30 @@ filegroup {
],
}
+// GN: //src/protozero/filtering:filter_util
+filegroup {
+ name: "perfetto_src_protozero_filtering_filter_util",
+ srcs: [
+ "src/protozero/filtering/filter_util.cc",
+ ],
+}
+
+// GN: //src/protozero/filtering:message_filter
+filegroup {
+ name: "perfetto_src_protozero_filtering_message_filter",
+ srcs: [
+ "src/protozero/filtering/message_filter.cc",
+ ],
+}
+
// GN: //src/protozero/filtering:unittests
filegroup {
name: "perfetto_src_protozero_filtering_unittests",
srcs: [
"src/protozero/filtering/filter_bytecode_generator_unittest.cc",
"src/protozero/filtering/filter_bytecode_parser_unittest.cc",
+ "src/protozero/filtering/filter_util_unittest.cc",
+ "src/protozero/filtering/message_filter_unittest.cc",
"src/protozero/filtering/message_tokenizer_unittest.cc",
],
}
@@ -8922,7 +8815,6 @@ cc_library_static {
":perfetto_protos_perfetto_config_power_lite_gen",
":perfetto_protos_perfetto_config_process_stats_lite_gen",
":perfetto_protos_perfetto_config_profiling_lite_gen",
- ":perfetto_protos_perfetto_config_proto_filter_lite_gen",
":perfetto_protos_perfetto_config_sys_stats_lite_gen",
":perfetto_protos_perfetto_config_track_event_lite_gen",
":perfetto_protos_perfetto_trace_android_lite_gen",
@@ -8956,7 +8848,6 @@ cc_library_static {
"perfetto_protos_perfetto_config_power_lite_gen_headers",
"perfetto_protos_perfetto_config_process_stats_lite_gen_headers",
"perfetto_protos_perfetto_config_profiling_lite_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_lite_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_lite_gen_headers",
"perfetto_protos_perfetto_config_track_event_lite_gen_headers",
"perfetto_protos_perfetto_trace_android_lite_gen_headers",
@@ -8986,7 +8877,6 @@ cc_library_static {
"perfetto_protos_perfetto_config_power_lite_gen_headers",
"perfetto_protos_perfetto_config_process_stats_lite_gen_headers",
"perfetto_protos_perfetto_config_profiling_lite_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_lite_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_lite_gen_headers",
"perfetto_protos_perfetto_config_track_event_lite_gen_headers",
"perfetto_protos_perfetto_trace_android_lite_gen_headers",
@@ -9071,9 +8961,6 @@ cc_test {
":perfetto_protos_perfetto_config_profiling_cpp_gen",
":perfetto_protos_perfetto_config_profiling_lite_gen",
":perfetto_protos_perfetto_config_profiling_zero_gen",
- ":perfetto_protos_perfetto_config_proto_filter_cpp_gen",
- ":perfetto_protos_perfetto_config_proto_filter_lite_gen",
- ":perfetto_protos_perfetto_config_proto_filter_zero_gen",
":perfetto_protos_perfetto_config_sys_stats_cpp_gen",
":perfetto_protos_perfetto_config_sys_stats_lite_gen",
":perfetto_protos_perfetto_config_sys_stats_zero_gen",
@@ -9176,8 +9063,11 @@ cc_test {
":perfetto_src_profiling_symbolizer_symbolizer",
":perfetto_src_profiling_symbolizer_unittests",
":perfetto_src_profiling_unittests",
+ ":perfetto_src_protozero_filtering_bytecode_common",
":perfetto_src_protozero_filtering_bytecode_generator",
":perfetto_src_protozero_filtering_bytecode_parser",
+ ":perfetto_src_protozero_filtering_filter_util",
+ ":perfetto_src_protozero_filtering_message_filter",
":perfetto_src_protozero_filtering_unittests",
":perfetto_src_protozero_protozero",
":perfetto_src_protozero_testing_messages_cpp_gen",
@@ -9271,6 +9161,7 @@ cc_test {
"libbase",
"liblog",
"libprocinfo",
+ "libprotobuf-cpp-full",
"libprotobuf-cpp-lite",
"libsqlite",
"libunwindstack",
@@ -9314,9 +9205,6 @@ cc_test {
"perfetto_protos_perfetto_config_profiling_cpp_gen_headers",
"perfetto_protos_perfetto_config_profiling_lite_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_lite_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_lite_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
@@ -9459,7 +9347,6 @@ cc_binary {
":perfetto_protos_perfetto_config_power_zero_gen",
":perfetto_protos_perfetto_config_process_stats_zero_gen",
":perfetto_protos_perfetto_config_profiling_zero_gen",
- ":perfetto_protos_perfetto_config_proto_filter_zero_gen",
":perfetto_protos_perfetto_config_sys_stats_zero_gen",
":perfetto_protos_perfetto_config_track_event_zero_gen",
":perfetto_protos_perfetto_config_zero_gen",
@@ -9519,7 +9406,6 @@ cc_binary {
"perfetto_protos_perfetto_config_power_zero_gen_headers",
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_zero_gen_headers",
"perfetto_protos_perfetto_config_zero_gen_headers",
@@ -9609,7 +9495,6 @@ cc_binary_host {
":perfetto_protos_perfetto_config_power_zero_gen",
":perfetto_protos_perfetto_config_process_stats_zero_gen",
":perfetto_protos_perfetto_config_profiling_zero_gen",
- ":perfetto_protos_perfetto_config_proto_filter_zero_gen",
":perfetto_protos_perfetto_config_sys_stats_zero_gen",
":perfetto_protos_perfetto_config_track_event_zero_gen",
":perfetto_protos_perfetto_config_zero_gen",
@@ -9676,7 +9561,6 @@ cc_binary_host {
"perfetto_protos_perfetto_config_power_zero_gen_headers",
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_zero_gen_headers",
"perfetto_protos_perfetto_config_zero_gen_headers",
@@ -9768,8 +9652,6 @@ cc_binary {
":perfetto_protos_perfetto_config_process_stats_zero_gen",
":perfetto_protos_perfetto_config_profiling_cpp_gen",
":perfetto_protos_perfetto_config_profiling_zero_gen",
- ":perfetto_protos_perfetto_config_proto_filter_cpp_gen",
- ":perfetto_protos_perfetto_config_proto_filter_zero_gen",
":perfetto_protos_perfetto_config_sys_stats_cpp_gen",
":perfetto_protos_perfetto_config_sys_stats_zero_gen",
":perfetto_protos_perfetto_config_track_event_cpp_gen",
@@ -9816,6 +9698,9 @@ cc_binary {
":perfetto_src_profiling_perf_regs_parsing",
":perfetto_src_profiling_perf_traced_perf_main",
":perfetto_src_profiling_perf_unwinding",
+ ":perfetto_src_protozero_filtering_bytecode_common",
+ ":perfetto_src_protozero_filtering_bytecode_parser",
+ ":perfetto_src_protozero_filtering_message_filter",
":perfetto_src_protozero_protozero",
":perfetto_src_traced_probes_ftrace_ftrace_procfs",
":perfetto_src_traced_probes_packages_list_packages_list_parser",
@@ -9855,8 +9740,6 @@ cc_binary {
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_cpp_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_cpp_gen_headers",
@@ -9950,8 +9833,6 @@ cc_binary {
":perfetto_protos_perfetto_config_process_stats_zero_gen",
":perfetto_protos_perfetto_config_profiling_cpp_gen",
":perfetto_protos_perfetto_config_profiling_zero_gen",
- ":perfetto_protos_perfetto_config_proto_filter_cpp_gen",
- ":perfetto_protos_perfetto_config_proto_filter_zero_gen",
":perfetto_protos_perfetto_config_sys_stats_cpp_gen",
":perfetto_protos_perfetto_config_sys_stats_zero_gen",
":perfetto_protos_perfetto_config_track_event_cpp_gen",
@@ -10017,8 +9898,6 @@ cc_binary {
"perfetto_protos_perfetto_config_process_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_profiling_cpp_gen_headers",
"perfetto_protos_perfetto_config_profiling_zero_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers",
- "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers",
"perfetto_protos_perfetto_config_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_config_track_event_cpp_gen_headers",
diff --git a/BUILD b/BUILD
index 87fff6aaa..0e006ec12 100644
--- a/BUILD
+++ b/BUILD
@@ -146,7 +146,6 @@ perfetto_cc_binary(
":protos_perfetto_config_power_zero",
":protos_perfetto_config_process_stats_zero",
":protos_perfetto_config_profiling_zero",
- ":protos_perfetto_config_proto_filter_zero",
":protos_perfetto_config_sys_stats_zero",
":protos_perfetto_config_track_event_cpp",
":protos_perfetto_config_track_event_zero",
@@ -179,6 +178,9 @@ perfetto_cc_library(
":src_android_stats_android_stats",
":src_android_stats_perfetto_atoms",
":src_kallsyms_kallsyms",
+ ":src_protozero_filtering_bytecode_common",
+ ":src_protozero_filtering_bytecode_parser",
+ ":src_protozero_filtering_message_filter",
":src_traced_probes_android_log_android_log",
":src_traced_probes_common_common",
":src_traced_probes_data_source",
@@ -241,8 +243,6 @@ perfetto_cc_library(
":protos_perfetto_config_process_stats_zero",
":protos_perfetto_config_profiling_cpp",
":protos_perfetto_config_profiling_zero",
- ":protos_perfetto_config_proto_filter_cpp",
- ":protos_perfetto_config_proto_filter_zero",
":protos_perfetto_config_sys_stats_cpp",
":protos_perfetto_config_sys_stats_zero",
":protos_perfetto_config_track_event_cpp",
@@ -776,6 +776,33 @@ filegroup(
],
)
+# GN target: //src/protozero/filtering:bytecode_common
+filegroup(
+ name = "src_protozero_filtering_bytecode_common",
+ srcs = [
+ "src/protozero/filtering/filter_bytecode_common.h",
+ ],
+)
+
+# GN target: //src/protozero/filtering:bytecode_parser
+filegroup(
+ name = "src_protozero_filtering_bytecode_parser",
+ srcs = [
+ "src/protozero/filtering/filter_bytecode_parser.cc",
+ "src/protozero/filtering/filter_bytecode_parser.h",
+ ],
+)
+
+# GN target: //src/protozero/filtering:message_filter
+filegroup(
+ name = "src_protozero_filtering_message_filter",
+ srcs = [
+ "src/protozero/filtering/message_filter.cc",
+ "src/protozero/filtering/message_filter.h",
+ "src/protozero/filtering/message_tokenizer.h",
+ ],
+)
+
# GN target: //src/trace_processor/analysis:analysis
filegroup(
name = "src_trace_processor_analysis_analysis",
@@ -1858,7 +1885,6 @@ perfetto_cc_protocpp_library(
":protos_perfetto_config_power_cpp",
":protos_perfetto_config_process_stats_cpp",
":protos_perfetto_config_profiling_cpp",
- ":protos_perfetto_config_proto_filter_cpp",
":protos_perfetto_config_protos",
":protos_perfetto_config_sys_stats_cpp",
":protos_perfetto_config_track_event_cpp",
@@ -2181,41 +2207,6 @@ perfetto_cc_protozero_library(
],
)
-# GN target: //protos/perfetto/config:proto_filter_cpp
-perfetto_cc_protocpp_library(
- name = "protos_perfetto_config_proto_filter_cpp",
- deps = [
- ":protos_perfetto_config_proto_filter_protos",
- ],
-)
-
-# GN target: //protos/perfetto/config:proto_filter_lite
-perfetto_cc_proto_library(
- name = "protos_perfetto_config_proto_filter_lite",
- deps = [
- ":protos_perfetto_config_proto_filter_protos",
- ],
-)
-
-# GN target: //protos/perfetto/config:proto_filter_zero
-perfetto_proto_library(
- name = "protos_perfetto_config_proto_filter_protos",
- srcs = [
- "protos/perfetto/config/proto_filter.proto",
- ],
- visibility = [
- PERFETTO_CONFIG.proto_library_visibility,
- ],
-)
-
-# GN target: //protos/perfetto/config:proto_filter_zero
-perfetto_cc_protozero_library(
- name = "protos_perfetto_config_proto_filter_zero",
- deps = [
- ":protos_perfetto_config_proto_filter_protos",
- ],
-)
-
# GN target: //protos/perfetto/config:zero
perfetto_proto_library(
name = "protos_perfetto_config_protos",
@@ -2238,7 +2229,6 @@ perfetto_proto_library(
":protos_perfetto_config_power_protos",
":protos_perfetto_config_process_stats_protos",
":protos_perfetto_config_profiling_protos",
- ":protos_perfetto_config_proto_filter_protos",
":protos_perfetto_config_sys_stats_protos",
":protos_perfetto_config_track_event_protos",
],
@@ -2332,7 +2322,6 @@ perfetto_cc_protozero_library(
":protos_perfetto_config_power_zero",
":protos_perfetto_config_process_stats_zero",
":protos_perfetto_config_profiling_zero",
- ":protos_perfetto_config_proto_filter_zero",
":protos_perfetto_config_protos",
":protos_perfetto_config_sys_stats_zero",
":protos_perfetto_config_track_event_zero",
@@ -2353,7 +2342,6 @@ perfetto_cc_protocpp_library(
":protos_perfetto_config_power_cpp",
":protos_perfetto_config_process_stats_cpp",
":protos_perfetto_config_profiling_cpp",
- ":protos_perfetto_config_proto_filter_cpp",
":protos_perfetto_config_sys_stats_cpp",
":protos_perfetto_config_track_event_cpp",
":protos_perfetto_ipc_protos",
@@ -2374,7 +2362,6 @@ perfetto_cc_ipc_library(
":protos_perfetto_config_power_cpp",
":protos_perfetto_config_process_stats_cpp",
":protos_perfetto_config_profiling_cpp",
- ":protos_perfetto_config_proto_filter_cpp",
":protos_perfetto_config_sys_stats_cpp",
":protos_perfetto_config_track_event_cpp",
":protos_perfetto_ipc_cpp",
@@ -2402,7 +2389,6 @@ perfetto_proto_library(
":protos_perfetto_config_power_protos",
":protos_perfetto_config_process_stats_protos",
":protos_perfetto_config_profiling_protos",
- ":protos_perfetto_config_proto_filter_protos",
":protos_perfetto_config_protos",
":protos_perfetto_config_sys_stats_protos",
":protos_perfetto_config_track_event_protos",
@@ -2827,7 +2813,6 @@ perfetto_proto_library(
":protos_perfetto_config_power_protos",
":protos_perfetto_config_process_stats_protos",
":protos_perfetto_config_profiling_protos",
- ":protos_perfetto_config_proto_filter_protos",
":protos_perfetto_config_protos",
":protos_perfetto_config_sys_stats_protos",
":protos_perfetto_config_track_event_protos",
@@ -2847,7 +2832,6 @@ perfetto_cc_protozero_library(
":protos_perfetto_config_power_zero",
":protos_perfetto_config_process_stats_zero",
":protos_perfetto_config_profiling_zero",
- ":protos_perfetto_config_proto_filter_zero",
":protos_perfetto_config_sys_stats_zero",
":protos_perfetto_config_track_event_zero",
":protos_perfetto_config_zero",
@@ -2887,7 +2871,6 @@ perfetto_proto_library(
":protos_perfetto_config_power_protos",
":protos_perfetto_config_process_stats_protos",
":protos_perfetto_config_profiling_protos",
- ":protos_perfetto_config_proto_filter_protos",
":protos_perfetto_config_protos",
":protos_perfetto_config_sys_stats_protos",
":protos_perfetto_config_track_event_protos",
@@ -2921,7 +2904,6 @@ perfetto_cc_protozero_library(
":protos_perfetto_config_power_zero",
":protos_perfetto_config_process_stats_zero",
":protos_perfetto_config_profiling_zero",
- ":protos_perfetto_config_proto_filter_zero",
":protos_perfetto_config_sys_stats_zero",
":protos_perfetto_config_track_event_zero",
":protos_perfetto_config_zero",
@@ -3323,6 +3305,9 @@ perfetto_cc_library(
srcs = [
":src_android_stats_android_stats",
":src_android_stats_perfetto_atoms",
+ ":src_protozero_filtering_bytecode_common",
+ ":src_protozero_filtering_bytecode_parser",
+ ":src_protozero_filtering_message_filter",
":src_tracing_client_api_without_backends",
":src_tracing_common",
":src_tracing_core_core",
@@ -3371,8 +3356,6 @@ perfetto_cc_library(
":protos_perfetto_config_process_stats_zero",
":protos_perfetto_config_profiling_cpp",
":protos_perfetto_config_profiling_zero",
- ":protos_perfetto_config_proto_filter_cpp",
- ":protos_perfetto_config_proto_filter_zero",
":protos_perfetto_config_sys_stats_cpp",
":protos_perfetto_config_sys_stats_zero",
":protos_perfetto_config_track_event_cpp",
@@ -3451,8 +3434,6 @@ perfetto_cc_binary(
":protos_perfetto_config_process_stats_zero",
":protos_perfetto_config_profiling_cpp",
":protos_perfetto_config_profiling_zero",
- ":protos_perfetto_config_proto_filter_cpp",
- ":protos_perfetto_config_proto_filter_zero",
":protos_perfetto_config_sys_stats_cpp",
":protos_perfetto_config_sys_stats_zero",
":protos_perfetto_config_track_event_cpp",
@@ -3530,7 +3511,6 @@ perfetto_cc_library(
":protos_perfetto_config_power_zero",
":protos_perfetto_config_process_stats_zero",
":protos_perfetto_config_profiling_zero",
- ":protos_perfetto_config_proto_filter_zero",
":protos_perfetto_config_sys_stats_zero",
":protos_perfetto_config_track_event_zero",
":protos_perfetto_config_zero",
@@ -3622,7 +3602,6 @@ perfetto_cc_binary(
":protos_perfetto_config_power_zero",
":protos_perfetto_config_process_stats_zero",
":protos_perfetto_config_profiling_zero",
- ":protos_perfetto_config_proto_filter_zero",
":protos_perfetto_config_sys_stats_zero",
":protos_perfetto_config_track_event_zero",
":protos_perfetto_config_zero",
@@ -3724,7 +3703,6 @@ perfetto_cc_library(
":protos_perfetto_config_power_zero",
":protos_perfetto_config_process_stats_zero",
":protos_perfetto_config_profiling_zero",
- ":protos_perfetto_config_proto_filter_zero",
":protos_perfetto_config_sys_stats_zero",
":protos_perfetto_config_track_event_zero",
":protos_perfetto_config_zero",
@@ -3804,7 +3782,6 @@ perfetto_cc_binary(
":protos_perfetto_config_power_zero",
":protos_perfetto_config_process_stats_zero",
":protos_perfetto_config_profiling_zero",
- ":protos_perfetto_config_proto_filter_zero",
":protos_perfetto_config_sys_stats_zero",
":protos_perfetto_config_track_event_zero",
":protos_perfetto_config_zero",
diff --git a/gn/BUILD.gn b/gn/BUILD.gn
index a297a845b..1f799ade3 100644
--- a/gn/BUILD.gn
+++ b/gn/BUILD.gn
@@ -195,7 +195,9 @@ group("gtest_main") {
protobuf_full_deps_allowlist = [
"../src/ipc/protoc_plugin:*",
"../src/protozero/protoc_plugin:*",
+ "../src/protozero/filtering:filter_util",
"../src/trace_processor:trace_processor_shell",
+ "../src/protozero/filtering:filter_util",
"../tools/*",
]
diff --git a/gn/perfetto_benchmarks.gni b/gn/perfetto_benchmarks.gni
index 2bde3766f..e1540649c 100644
--- a/gn/perfetto_benchmarks.gni
+++ b/gn/perfetto_benchmarks.gni
@@ -18,6 +18,7 @@ perfetto_benchmarks_targets = [
"gn:default_deps",
"src/base:benchmarks",
"src/protozero:benchmarks",
+ "src/protozero/filtering:benchmarks",
"src/trace_processor/sqlite:benchmarks",
"src/trace_processor/containers:benchmarks",
"src/trace_processor/tables:benchmarks",
diff --git a/gn/perfetto_fuzzers.gni b/gn/perfetto_fuzzers.gni
index 049d5251b..5874f8e14 100644
--- a/gn/perfetto_fuzzers.gni
+++ b/gn/perfetto_fuzzers.gni
@@ -19,6 +19,7 @@ perfetto_fuzzers_targets = [
"src/ipc:buffered_frame_deserializer_fuzzer",
"src/protozero:protozero_decoder_fuzzer",
"src/protozero/filtering:protozero_bytecode_parser_fuzzer",
+ "src/protozero/filtering:protozero_message_filter_fuzzer",
"src/tracing/core:packet_stream_validator_fuzzer",
"src/trace_processor:trace_processor_fuzzer",
"src/traced/probes/ftrace:cpu_reader_fuzzer",
diff --git a/include/perfetto/ext/tracing/core/slice.h b/include/perfetto/ext/tracing/core/slice.h
index 4ce66fdea..063043bdc 100644
--- a/include/perfetto/ext/tracing/core/slice.h
+++ b/include/perfetto/ext/tracing/core/slice.h
@@ -44,6 +44,14 @@ struct Slice {
return slice;
}
+ static Slice TakeOwnership(std::unique_ptr<uint8_t[]> buf, size_t size) {
+ Slice slice;
+ slice.own_data_ = std::move(buf);
+ slice.start = &slice.own_data_[0];
+ slice.size = size;
+ return slice;
+ }
+
uint8_t* own_data() {
PERFETTO_DCHECK(own_data_);
return own_data_.get();
diff --git a/include/perfetto/protozero/proto_utils.h b/include/perfetto/protozero/proto_utils.h
index 841042b17..6b195cb1e 100644
--- a/include/perfetto/protozero/proto_utils.h
+++ b/include/perfetto/protozero/proto_utils.h
@@ -201,12 +201,20 @@ inline uint8_t* WriteVarInt(T value, uint8_t* target) {
// used to backfill fixed-size reservations for the length field using a
// non-canonical varint encoding (e.g. \x81\x80\x80\x00 instead of \x01).
// See https://github.com/google/protobuf/issues/1530.
-// In particular, this is used for nested messages. The size of a nested message
-// is not known until all its field have been written. |kMessageLengthFieldSize|
-// bytes are reserved to encode the size field and backfilled at the end.
-inline void WriteRedundantVarInt(uint32_t value, uint8_t* buf) {
- for (size_t i = 0; i < kMessageLengthFieldSize; ++i) {
- const uint8_t msb = (i < kMessageLengthFieldSize - 1) ? 0x80 : 0;
+// This is used mainly in two cases:
+// 1) At trace writing time, when starting a nested messages. The size of a
+// nested message is not known until all its field have been written.
+// |kMessageLengthFieldSize| bytes are reserved to encode the size field and
+// backfilled at the end.
+// 2) When rewriting a message at trace filtering time, in protozero/filtering.
+// At that point we know only the upper bound of the length (a filtered
+// message is <= the original one) and we backfill after the message has been
+// filtered.
+inline void WriteRedundantVarInt(uint32_t value,
+ uint8_t* buf,
+ size_t size = kMessageLengthFieldSize) {
+ for (size_t i = 0; i < size; ++i) {
+ const uint8_t msb = (i < size - 1) ? 0x80 : 0;
buf[i] = static_cast<uint8_t>(value) | msb;
value >>= 7;
}
diff --git a/protos/perfetto/common/trace_stats.proto b/protos/perfetto/common/trace_stats.proto
index e925ad460..c6a9fbfa1 100644
--- a/protos/perfetto/common/trace_stats.proto
+++ b/protos/perfetto/common/trace_stats.proto
@@ -158,4 +158,13 @@ message TraceStats {
// Packets that failed validation of the TrustedPacket. If this is > 0, there
// is a bug in the producer.
optional uint64 invalid_packets = 10;
+
+ // This is set only when the TraceConfig specifies a TraceFilter.
+ message FilterStats {
+ optional uint64 input_packets = 1;
+ optional uint64 input_bytes = 2;
+ optional uint64 output_bytes = 3;
+ optional uint64 errors = 4;
+ }
+ optional FilterStats filter_stats = 11;
}
diff --git a/protos/perfetto/config/BUILD.gn b/protos/perfetto/config/BUILD.gn
index 60bcaf6a8..de9c1f542 100644
--- a/protos/perfetto/config/BUILD.gn
+++ b/protos/perfetto/config/BUILD.gn
@@ -20,7 +20,6 @@ import("../../../gn/proto_library.gni")
perfetto_proto_library("@TYPE@") {
deps = [
- ":proto_filter_@TYPE@",
"../common:@TYPE@",
"android:@TYPE@",
"ftrace:@TYPE@",
@@ -63,7 +62,3 @@ perfetto_proto_library("perfetto_config_descriptor") {
generate_descriptor = "perfetto_config.descriptor"
sources = [ "perfetto_config.proto" ]
}
-
-perfetto_proto_library("proto_filter_@TYPE@") {
- sources = [ "proto_filter.proto" ]
-}
diff --git a/protos/perfetto/config/perfetto_config.proto b/protos/perfetto/config/perfetto_config.proto
index d088e9b84..5b0f2b692 100644
--- a/protos/perfetto/config/perfetto_config.proto
+++ b/protos/perfetto/config/perfetto_config.proto
@@ -1462,7 +1462,7 @@ message DataSourceConfig {
// It contains the general config for the logging buffer(s) and the configs for
// all the data source being enabled.
//
-// Next id: 31.
+// Next id: 33.
message TraceConfig {
message BufferConfig {
optional uint32 size_kb = 1;
@@ -1857,6 +1857,15 @@ message TraceConfig {
// Alternative encoding of trace_uuid as two int64s.
optional int64 trace_uuid_msb = 27;
optional int64 trace_uuid_lsb = 28;
+
+ // When set applies a post-filter to the trace contents using the filter
+ // provided. The filter is applied at ReadBuffers() time and works both in the
+ // case of IPC readback and write_into_file. This filter can be generated
+ // using `tools/proto_filter -s schema.proto -F filter_out.bytes` or
+ // `-T filter_out.escaped_string` (for .pbtx).
+ // Introduced in Android S. See go/trace-filtering for design.
+ message TraceFilter { optional bytes bytecode = 1; }
+ optional TraceFilter trace_filter = 32;
}
// End of protos/perfetto/config/trace_config.proto
diff --git a/protos/perfetto/config/proto_filter.proto b/protos/perfetto/config/proto_filter.proto
deleted file mode 100644
index deec6935a..000000000
--- a/protos/perfetto/config/proto_filter.proto
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (C) 2021 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-syntax = "proto2";
-
-package perfetto.protos;
-
-// This proto wraps a bytecode that describes how to filter proto messages.
-// For the full specs of the filter bytecode see go/trace-filtering.
-// In short:
-// - A filter is a sequence of uint32 words.
-// - Each word is (immediate value << 3) | (3-bit OPCODE)
-// - Each group of words (until END_OF_MESSAGE) defines field filters for a
-// message. The first message is the root message where the filtering starts
-// (typically perfetto.protos.Trace).
-message ProtoFilter {
- enum Opcode {
- // The immediate value is 0 in this case.
- FILTER_OPCODE_END_OF_MESSAGE = 0;
-
- // The immediate value is the id of the allowed field.
- FILTER_OPCODE_SIMPLE_FIELD = 1;
-
- // The immediate value is the start of the range. The next word (without
- // any shifting) is the length of the range.
- FILTER_OPCODE_SIMPLE_FIELD_RANGE = 2;
-
- // The immediate value is the id of the allowed field. The next word
- // (without any shifting) is the index of the filter that should be used to
- // recurse into the nested message.
- FILTER_OPCODE_NESTED_FIELD = 3;
- }
- repeated uint32 bytecode = 1 [packed = true];
-}
diff --git a/protos/perfetto/config/trace_config.proto b/protos/perfetto/config/trace_config.proto
index 57677d3fe..dbc1babda 100644
--- a/protos/perfetto/config/trace_config.proto
+++ b/protos/perfetto/config/trace_config.proto
@@ -26,7 +26,7 @@ package perfetto.protos;
// It contains the general config for the logging buffer(s) and the configs for
// all the data source being enabled.
//
-// Next id: 31.
+// Next id: 33.
message TraceConfig {
message BufferConfig {
optional uint32 size_kb = 1;
@@ -421,4 +421,13 @@ message TraceConfig {
// Alternative encoding of trace_uuid as two int64s.
optional int64 trace_uuid_msb = 27;
optional int64 trace_uuid_lsb = 28;
+
+ // When set applies a post-filter to the trace contents using the filter
+ // provided. The filter is applied at ReadBuffers() time and works both in the
+ // case of IPC readback and write_into_file. This filter can be generated
+ // using `tools/proto_filter -s schema.proto -F filter_out.bytes` or
+ // `-T filter_out.escaped_string` (for .pbtx).
+ // Introduced in Android S. See go/trace-filtering for design.
+ message TraceFilter { optional bytes bytecode = 1; }
+ optional TraceFilter trace_filter = 32;
}
diff --git a/protos/perfetto/trace/perfetto_trace.proto b/protos/perfetto/trace/perfetto_trace.proto
index 36313affd..402fae891 100644
--- a/protos/perfetto/trace/perfetto_trace.proto
+++ b/protos/perfetto/trace/perfetto_trace.proto
@@ -1462,7 +1462,7 @@ message DataSourceConfig {
// It contains the general config for the logging buffer(s) and the configs for
// all the data source being enabled.
//
-// Next id: 31.
+// Next id: 33.
message TraceConfig {
message BufferConfig {
optional uint32 size_kb = 1;
@@ -1857,6 +1857,15 @@ message TraceConfig {
// Alternative encoding of trace_uuid as two int64s.
optional int64 trace_uuid_msb = 27;
optional int64 trace_uuid_lsb = 28;
+
+ // When set applies a post-filter to the trace contents using the filter
+ // provided. The filter is applied at ReadBuffers() time and works both in the
+ // case of IPC readback and write_into_file. This filter can be generated
+ // using `tools/proto_filter -s schema.proto -F filter_out.bytes` or
+ // `-T filter_out.escaped_string` (for .pbtx).
+ // Introduced in Android S. See go/trace-filtering for design.
+ message TraceFilter { optional bytes bytecode = 1; }
+ optional TraceFilter trace_filter = 32;
}
// End of protos/perfetto/config/trace_config.proto
@@ -2003,6 +2012,15 @@ message TraceStats {
// Packets that failed validation of the TrustedPacket. If this is > 0, there
// is a bug in the producer.
optional uint64 invalid_packets = 10;
+
+ // This is set only when the TraceConfig specifies a TraceFilter.
+ message FilterStats {
+ optional uint64 input_packets = 1;
+ optional uint64 input_bytes = 2;
+ optional uint64 output_bytes = 3;
+ optional uint64 errors = 4;
+ }
+ optional FilterStats filter_stats = 11;
}
// End of protos/perfetto/common/trace_stats.proto
diff --git a/src/android_stats/perfetto_atoms.h b/src/android_stats/perfetto_atoms.h
index cf73d5f65..3731c2e76 100644
--- a/src/android_stats/perfetto_atoms.h
+++ b/src/android_stats/perfetto_atoms.h
@@ -68,6 +68,7 @@ enum class PerfettoStatsdAtom {
kTracedEnableTracingOom = 34,
kTracedEnableTracingUnknown = 35,
kTracedStartTracingInvalidSessionState = 36,
+ kTracedEnableTracingInvalidFilter = 47,
// Checkpoints inside perfetto_cmd after tracing has finished.
kOnTracingDisabled = 4,
diff --git a/src/perfetto_cmd/pbtxt_to_pb.cc b/src/perfetto_cmd/pbtxt_to_pb.cc
index f4f0c00a1..b78d93829 100644
--- a/src/perfetto_cmd/pbtxt_to_pb.cc
+++ b/src/perfetto_cmd/pbtxt_to_pb.cc
@@ -47,12 +47,20 @@ using protos::gen::FileDescriptorSet;
namespace {
+constexpr bool IsOct(char c) {
+ return (c >= '0' && c <= '7');
+}
+
+constexpr bool IsDigit(char c) {
+ return (c >= '0' && c <= '9');
+}
+
constexpr bool IsIdentifierStart(char c) {
return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || c == '_';
}
constexpr bool IsIdentifierBody(char c) {
- return IsIdentifierStart(c) || isdigit(c);
+ return IsIdentifierStart(c) || IsDigit(c);
}
const char* FieldToTypeName(const FieldDescriptorProto* field) {
@@ -150,19 +158,22 @@ class ParserDelegate {
}
void NumericField(Token key, Token value) {
- const FieldDescriptorProto* field = FindFieldByName(
- key, value,
- {
- FieldDescriptorProto::TYPE_UINT64,
- FieldDescriptorProto::TYPE_UINT32, FieldDescriptorProto::TYPE_INT64,
- FieldDescriptorProto::TYPE_SINT64, FieldDescriptorProto::TYPE_INT32,
- FieldDescriptorProto::TYPE_SINT32,
- FieldDescriptorProto::TYPE_FIXED64,
- FieldDescriptorProto::TYPE_SFIXED64,
- FieldDescriptorProto::TYPE_FIXED32,
- FieldDescriptorProto::TYPE_SFIXED32,
- FieldDescriptorProto::TYPE_DOUBLE, FieldDescriptorProto::TYPE_FLOAT,
- });
+ const FieldDescriptorProto* field =
+ FindFieldByName(key, value,
+ {
+ FieldDescriptorProto::TYPE_UINT64,
+ FieldDescriptorProto::TYPE_UINT32,
+ FieldDescriptorProto::TYPE_INT64,
+ FieldDescriptorProto::TYPE_SINT64,
+ FieldDescriptorProto::TYPE_INT32,
+ FieldDescriptorProto::TYPE_SINT32,
+ FieldDescriptorProto::TYPE_FIXED64,
+ FieldDescriptorProto::TYPE_SFIXED64,
+ FieldDescriptorProto::TYPE_FIXED32,
+ FieldDescriptorProto::TYPE_SFIXED32,
+ FieldDescriptorProto::TYPE_DOUBLE,
+ FieldDescriptorProto::TYPE_FLOAT,
+ });
if (!field)
return;
const auto& field_type = field->type();
@@ -202,11 +213,12 @@ class ParserDelegate {
}
void StringField(Token key, Token value) {
- const FieldDescriptorProto* field = FindFieldByName(
- key, value,
- {
- FieldDescriptorProto::TYPE_STRING, FieldDescriptorProto::TYPE_BYTES,
- });
+ const FieldDescriptorProto* field =
+ FindFieldByName(key, value,
+ {
+ FieldDescriptorProto::TYPE_STRING,
+ FieldDescriptorProto::TYPE_BYTES,
+ });
if (!field)
return;
uint32_t field_id = static_cast<uint32_t>(field->number());
@@ -217,15 +229,16 @@ class ParserDelegate {
std::unique_ptr<char, base::FreeDeleter> s(
static_cast<char*>(malloc(value.size())));
size_t j = 0;
+ const char* const txt = value.txt.data();
for (size_t i = 0; i < value.size(); i++) {
- char c = value.txt.data()[i];
+ char c = txt[i];
if (c == '\\') {
if (i + 1 >= value.size()) {
// This should be caught by the lexer.
PERFETTO_FATAL("Escape at end of string.");
return;
}
- char next = value.txt.data()[++i];
+ char next = txt[++i];
switch (next) {
case '\\':
case '\'':
@@ -254,6 +267,44 @@ class ParserDelegate {
case 'v':
s.get()[j++] = '\v';
break;
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9': {
+ // Cases 8 and 9 are not really required and are only added for the
+ // sake of error reporting.
+ bool oct_err = false;
+ if (i + 2 >= value.size() || !IsOct(txt[i + 1]) ||
+ !IsOct(txt[i + 2])) {
+ oct_err = true;
+ } else {
+ char buf[4]{next, txt[++i], txt[++i], '\0'};
+ auto octval = base::CStringToUInt32(buf, 8);
+ if (!octval.has_value() || *octval > 0xff) {
+ oct_err = true;
+ } else {
+ s.get()[j++] = static_cast<char>(static_cast<uint8_t>(*octval));
+ }
+ }
+ if (oct_err) {
+ AddError(value,
+ "Malformed string escape in $k in proto $n on '$v'. "
+ "\\NNN escapes must be exactly three octal digits <= "
+ "\\377 (0xff).",
+ std::map<std::string, std::string>{
+ {"$k", key.ToStdString()},
+ {"$n", descriptor_name()},
+ {"$v", value.ToStdString()},
+ });
+ }
+ break;
+ }
default:
AddError(value,
"Unknown string escape in $k in "
@@ -273,11 +324,12 @@ class ParserDelegate {
}
void IdentifierField(Token key, Token value) {
- const FieldDescriptorProto* field = FindFieldByName(
- key, value,
- {
- FieldDescriptorProto::TYPE_BOOL, FieldDescriptorProto::TYPE_ENUM,
- });
+ const FieldDescriptorProto* field =
+ FindFieldByName(key, value,
+ {
+ FieldDescriptorProto::TYPE_BOOL,
+ FieldDescriptorProto::TYPE_ENUM,
+ });
if (!field)
return;
uint32_t field_id = static_cast<uint32_t>(field->number());
@@ -415,7 +467,8 @@ class ParserDelegate {
if (!field_descriptor) {
AddError(key, "No field named \"$n\" in proto $p",
{
- {"$n", field_name}, {"$p", descriptor_name()},
+ {"$n", field_name},
+ {"$p", descriptor_name()},
});
return nullptr;
}
@@ -549,7 +602,7 @@ void Parse(const std::string& input, ParserDelegate* delegate) {
state = kReadingStringValue;
continue;
}
- if (c == '-' || isdigit(c) || c == '.') {
+ if (c == '-' || IsDigit(c) || c == '.') {
state = kReadingNumericValue;
continue;
}
@@ -576,7 +629,7 @@ void Parse(const std::string& input, ParserDelegate* delegate) {
delegate->NumericField(key, value);
continue;
}
- if (isdigit(c) || c == '.')
+ if (IsDigit(c) || c == '.')
continue;
break;
diff --git a/src/perfetto_cmd/pbtxt_to_pb_unittest.cc b/src/perfetto_cmd/pbtxt_to_pb_unittest.cc
index 00dab16f0..251c8f07e 100644
--- a/src/perfetto_cmd/pbtxt_to_pb_unittest.cc
+++ b/src/perfetto_cmd/pbtxt_to_pb_unittest.cc
@@ -403,6 +403,7 @@ data_sources {
ftrace_events: "newline\nnewline"
ftrace_events: "\"quoted\""
ftrace_events: "\a\b\f\n\r\t\v\\\'\"\?"
+ ftrace_events: "\0127_\03422.\177"
}
}
}
@@ -417,6 +418,7 @@ data_sources {
EXPECT_THAT(events, Contains("newline\nnewline"));
EXPECT_THAT(events, Contains("\"quoted\""));
EXPECT_THAT(events, Contains("\a\b\f\n\r\t\v\\\'\"\?"));
+ EXPECT_THAT(events, Contains("\0127_\03422.\177"));
}
TEST(PbtxtToPb, UnknownField) {
diff --git a/src/protozero/filtering/BUILD.gn b/src/protozero/filtering/BUILD.gn
index 0ddfc3eff..b438929b4 100644
--- a/src/protozero/filtering/BUILD.gn
+++ b/src/protozero/filtering/BUILD.gn
@@ -17,15 +17,34 @@ import("../../../gn/perfetto_host_executable.gni")
import("../../../gn/proto_library.gni")
import("../../../gn/test.gni")
+source_set("message_filter") {
+ sources = [
+ "message_filter.cc",
+ "message_filter.h",
+ "message_tokenizer.h",
+ ]
+ deps = [
+ ":bytecode_parser",
+ "..:protozero",
+ "../../../gn:default_deps",
+ "../../base",
+ ]
+}
+
+source_set("bytecode_common") {
+ sources = [ "filter_bytecode_common.h" ]
+ deps = [ "../../../gn:default_deps" ]
+}
+
source_set("bytecode_parser") {
sources = [
"filter_bytecode_parser.cc",
"filter_bytecode_parser.h",
]
deps = [
+ ":bytecode_common",
"..:protozero",
"../../../gn:default_deps",
- "../../../protos/perfetto/config:proto_filter_zero",
"../../base",
]
}
@@ -36,9 +55,24 @@ source_set("bytecode_generator") {
"filter_bytecode_generator.h",
]
deps = [
+ ":bytecode_common",
+ "..:protozero",
+ "../../../gn:default_deps",
+ "../../base",
+ ]
+}
+
+source_set("filter_util") {
+ testonly = true
+ sources = [
+ "filter_util.cc",
+ "filter_util.h",
+ ]
+ deps = [
+ ":bytecode_generator",
"..:protozero",
"../../../gn:default_deps",
- "../../../protos/perfetto/config:proto_filter_zero",
+ "../../../gn:protobuf_full",
"../../base",
]
}
@@ -46,26 +80,57 @@ source_set("bytecode_generator") {
perfetto_unittest_source_set("unittests") {
testonly = true
deps = [
+ ":bytecode_common",
":bytecode_generator",
":bytecode_parser",
+ ":filter_util",
+ ":message_filter",
"..:protozero",
"../../../gn:default_deps",
"../../../gn:gtest_and_gmock",
- "../../../protos/perfetto/config:proto_filter_zero",
+ "../../../protos/perfetto/trace:lite",
"../../base",
"../../base:test_support",
]
sources = [
"filter_bytecode_generator_unittest.cc",
"filter_bytecode_parser_unittest.cc",
+ "filter_util_unittest.cc",
+ "message_filter_unittest.cc",
"message_tokenizer_unittest.cc",
]
}
+if (enable_perfetto_benchmarks) {
+ source_set("benchmarks") {
+ testonly = true
+ deps = [
+ ":message_filter",
+ "../../../gn:benchmark",
+ "../../../gn:default_deps",
+ "../../base",
+ "../../base:test_support",
+ ]
+ sources = [ "message_filter_benchmark.cc" ]
+ }
+}
+
perfetto_fuzzer_test("protozero_bytecode_parser_fuzzer") {
sources = [ "filter_bytecode_parser_fuzzer.cc" ]
deps = [
":bytecode_parser",
+ "..:protozero",
"../../../gn:default_deps",
+ "../../base",
+ ]
+}
+
+perfetto_fuzzer_test("protozero_message_filter_fuzzer") {
+ sources = [ "message_filter_fuzzer.cc" ]
+ deps = [
+ ":message_filter",
+ "..:protozero",
+ "../../../gn:default_deps",
+ "../../base",
]
}
diff --git a/src/protozero/filtering/filter_bytecode_common.h b/src/protozero/filtering/filter_bytecode_common.h
new file mode 100644
index 000000000..9c0e6f238
--- /dev/null
+++ b/src/protozero/filtering/filter_bytecode_common.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SRC_PROTOZERO_FILTERING_FILTER_BYTECODE_COMMON_H_
+#define SRC_PROTOZERO_FILTERING_FILTER_BYTECODE_COMMON_H_
+
+#include <stdint.h>
+
+namespace protozero {
+
+enum FilterOpcode : uint32_t {
+ // The immediate value is 0 in this case.
+ kFilterOpcode_EndOfMessage = 0,
+
+ // The immediate value is the id of the allowed field.
+ kFilterOpcode_SimpleField = 1,
+
+ // The immediate value is the start of the range. The next word (without
+ // any shifting) is the length of the range.
+ kFilterOpcode_SimpleFieldRange = 2,
+
+ // The immediate value is the id of the allowed field. The next word
+ // (without any shifting) is the index of the filter that should be used to
+ // recurse into the nested message.
+ kFilterOpcode_NestedField = 3,
+};
+} // namespace protozero
+
+#endif // SRC_PROTOZERO_FILTERING_FILTER_BYTECODE_COMMON_H_
diff --git a/src/protozero/filtering/filter_bytecode_generator.cc b/src/protozero/filtering/filter_bytecode_generator.cc
index df3a80e62..936286915 100644
--- a/src/protozero/filtering/filter_bytecode_generator.cc
+++ b/src/protozero/filtering/filter_bytecode_generator.cc
@@ -17,22 +17,20 @@
#include "src/protozero/filtering/filter_bytecode_generator.h"
#include "perfetto/base/logging.h"
+#include "perfetto/ext/base/hash.h"
#include "perfetto/protozero/packed_repeated_fields.h"
#include "perfetto/protozero/proto_utils.h"
#include "perfetto/protozero/scattered_heap_buffer.h"
-
-#include "protos/perfetto/config/proto_filter.pbzero.h"
+#include "src/protozero/filtering/filter_bytecode_common.h"
namespace protozero {
-using ProtoFilter = perfetto::protos::pbzero::ProtoFilter;
-
FilterBytecodeGenerator::FilterBytecodeGenerator() = default;
FilterBytecodeGenerator::~FilterBytecodeGenerator() = default;
void FilterBytecodeGenerator::EndMessage() {
endmessage_called_ = true;
- bytecode_.push_back(ProtoFilter::FILTER_OPCODE_END_OF_MESSAGE);
+ bytecode_.push_back(kFilterOpcode_EndOfMessage);
last_field_id_ = 0;
++num_messages_;
}
@@ -40,7 +38,7 @@ void FilterBytecodeGenerator::EndMessage() {
// Allows a simple field (varint, fixed32/64, string or bytes).
void FilterBytecodeGenerator::AddSimpleField(uint32_t field_id) {
PERFETTO_CHECK(field_id > last_field_id_);
- bytecode_.push_back(field_id << 3 | ProtoFilter::FILTER_OPCODE_SIMPLE_FIELD);
+ bytecode_.push_back(field_id << 3 | kFilterOpcode_SimpleField);
last_field_id_ = field_id;
endmessage_called_ = false;
}
@@ -52,8 +50,7 @@ void FilterBytecodeGenerator::AddSimpleFieldRange(uint32_t range_start,
uint32_t range_len) {
PERFETTO_CHECK(range_start > last_field_id_);
PERFETTO_CHECK(range_len > 0);
- bytecode_.push_back(range_start << 3 |
- ProtoFilter::FILTER_OPCODE_SIMPLE_FIELD_RANGE);
+ bytecode_.push_back(range_start << 3 | kFilterOpcode_SimpleFieldRange);
bytecode_.push_back(range_len);
last_field_id_ = range_start + range_len - 1;
endmessage_called_ = false;
@@ -67,26 +64,28 @@ void FilterBytecodeGenerator::AddSimpleFieldRange(uint32_t range_start,
void FilterBytecodeGenerator::AddNestedField(uint32_t field_id,
uint32_t message_index) {
PERFETTO_CHECK(field_id > last_field_id_);
- bytecode_.push_back(field_id << 3 | ProtoFilter::FILTER_OPCODE_NESTED_FIELD);
+ bytecode_.push_back(field_id << 3 | kFilterOpcode_NestedField);
bytecode_.push_back(message_index);
last_field_id_ = field_id;
max_msg_index_ = std::max(max_msg_index_, message_index);
endmessage_called_ = false;
}
-// Returns the proto-encoded bytes for a perfetto.protos.ProtoFilter message
-// (see proto_filter.proto). The returned string can be passed to
-// FilterBytecodeParser.Load().
+// Returns the bytes that can be used into TraceConfig.trace_filter.bytecode.
+// The returned bytecode is a binary buffer which consists of a sequence of
+// varints (the opcodes) and a checksum.
+// The returned string can be passed as-is to FilterBytecodeParser.Load().
std::string FilterBytecodeGenerator::Serialize() {
PERFETTO_CHECK(endmessage_called_);
PERFETTO_CHECK(max_msg_index_ < num_messages_);
protozero::PackedVarInt words;
- for (uint32_t word : bytecode_)
+ perfetto::base::Hash hasher;
+ for (uint32_t word : bytecode_) {
words.Append(word);
-
- protozero::HeapBuffered<ProtoFilter> filter;
- filter->set_bytecode(words);
- return filter.SerializeAsString();
+ hasher.Update(word);
+ }
+ words.Append(static_cast<uint32_t>(hasher.digest()));
+ return std::string(reinterpret_cast<const char*>(words.data()), words.size());
}
} // namespace protozero
diff --git a/src/protozero/filtering/filter_bytecode_generator.h b/src/protozero/filtering/filter_bytecode_generator.h
index 481d39fb4..809799c24 100644
--- a/src/protozero/filtering/filter_bytecode_generator.h
+++ b/src/protozero/filtering/filter_bytecode_generator.h
@@ -57,8 +57,8 @@ class FilterBytecodeGenerator {
// made).
void AddNestedField(uint32_t field_id, uint32_t message_index);
- // Returns the proto-encoded bytes for a perfetto.protos.ProtoFilter message
- // (see proto_filter.proto). The returned string can be passed to
+ // Returns the filter bytecode, which is a buffer containing a sequence of
+ // varints and a checksum. The returned string can be passed to
// FilterBytecodeParser.Load().
std::string Serialize();
diff --git a/src/protozero/filtering/filter_bytecode_generator_unittest.cc b/src/protozero/filtering/filter_bytecode_generator_unittest.cc
index 95181f578..650120b84 100644
--- a/src/protozero/filtering/filter_bytecode_generator_unittest.cc
+++ b/src/protozero/filtering/filter_bytecode_generator_unittest.cc
@@ -21,8 +21,6 @@
#include "src/protozero/filtering/filter_bytecode_generator.h"
#include "src/protozero/filtering/filter_bytecode_parser.h"
-#include "protos/perfetto/config/proto_filter.pbzero.h"
-
// This file tests the generator, assuming the parser is good.
// The parser is tested separately (without the generator) in
// filter_bytecode_parser_unittest.cc
diff --git a/src/protozero/filtering/filter_bytecode_parser.cc b/src/protozero/filtering/filter_bytecode_parser.cc
index 1a1d32fe8..39c8bb640 100644
--- a/src/protozero/filtering/filter_bytecode_parser.cc
+++ b/src/protozero/filtering/filter_bytecode_parser.cc
@@ -17,15 +17,18 @@
#include "src/protozero/filtering/filter_bytecode_parser.h"
#include "perfetto/base/logging.h"
+#include "perfetto/ext/base/hash.h"
#include "perfetto/protozero/packed_repeated_fields.h"
+#include "perfetto/protozero/proto_decoder.h"
#include "perfetto/protozero/proto_utils.h"
-
-#include "protos/perfetto/config/proto_filter.pbzero.h"
+#include "src/protozero/filtering/filter_bytecode_common.h"
namespace protozero {
void FilterBytecodeParser::Reset() {
+ bool suppress = suppress_logs_for_fuzzer_;
*this = FilterBytecodeParser();
+ suppress_logs_for_fuzzer_ = suppress;
}
bool FilterBytecodeParser::Load(const void* filter_data, size_t len) {
@@ -37,22 +40,37 @@ bool FilterBytecodeParser::Load(const void* filter_data, size_t len) {
return res;
}
-bool FilterBytecodeParser::LoadInternal(const uint8_t* filter_data,
+bool FilterBytecodeParser::LoadInternal(const uint8_t* bytecode_data,
size_t len) {
- using ProtoFilter = perfetto::protos::pbzero::ProtoFilter;
// First unpack the varints into a plain uint32 vector, so it's easy to
// iterate through them and look ahead.
- std::vector<uint32_t> bytecode;
- {
- ProtoFilter::Decoder dec(filter_data, len);
- bool packed_parse_err = false;
- bytecode.reserve(len); // An overestimation, but avoids reallocations.
- for (auto it = dec.bytecode(&packed_parse_err); it; ++it)
- bytecode.emplace_back(*it);
- if (packed_parse_err)
- return false;
+ std::vector<uint32_t> words;
+ bool packed_parse_err = false;
+ words.reserve(len); // An overestimation, but avoids reallocations.
+ using BytecodeDecoder =
+ PackedRepeatedFieldIterator<proto_utils::ProtoWireType::kVarInt,
+ uint32_t>;
+ for (BytecodeDecoder it(bytecode_data, len, &packed_parse_err); it; ++it)
+ words.emplace_back(*it);
+
+ if (packed_parse_err || words.empty())
+ return false;
+
+ perfetto::base::Hash hasher;
+ for (size_t i = 0; i < words.size() - 1; ++i)
+ hasher.Update(words[i]);
+
+ uint32_t expected_csum = static_cast<uint32_t>(hasher.digest());
+ if (expected_csum != words.back()) {
+ if (!suppress_logs_for_fuzzer_) {
+ PERFETTO_ELOG("Filter bytecode checksum failed. Expected: %x, actual: %x",
+ expected_csum, words.back());
+ }
+ return false;
}
+ words.pop_back(); // Pop the checksum.
+
// Temporay storage for each message. Cleared on every END_OF_MESSAGE.
std::vector<uint32_t> direct_indexed_fields;
std::vector<uint32_t> ranges;
@@ -73,26 +91,26 @@ bool FilterBytecodeParser::LoadInternal(const uint8_t* filter_data,
ranges.emplace_back(kAllowed | msg_id);
};
- for (size_t i = 0; i < bytecode.size(); ++i) {
- const uint32_t word = bytecode[i];
- const bool has_next_word = i < bytecode.size() - 1;
+ for (size_t i = 0; i < words.size(); ++i) {
+ const uint32_t word = words[i];
+ const bool has_next_word = i < words.size() - 1;
const uint32_t opcode = word & 0x7u;
const uint32_t field_id = word >> 3;
- if (opcode == ProtoFilter::FILTER_OPCODE_SIMPLE_FIELD ||
- opcode == ProtoFilter::FILTER_OPCODE_NESTED_FIELD) {
- if (field_id == 0) {
- PERFETTO_DLOG("bytecode error @ word %zu, invalid field id (0)", i);
- return false;
- }
+ if (field_id == 0 && opcode != kFilterOpcode_EndOfMessage) {
+ PERFETTO_DLOG("bytecode error @ word %zu, invalid field id (0)", i);
+ return false;
+ }
+ if (opcode == kFilterOpcode_SimpleField ||
+ opcode == kFilterOpcode_NestedField) {
// Field words are organized as follow:
// MSB: 1 if allowed, 0 if not allowed.
// Remaining bits:
// Message index in the case of nested (non-simple) messages.
// 0x7f..f in the case of simple messages.
uint32_t msg_id;
- if (opcode == ProtoFilter::FILTER_OPCODE_SIMPLE_FIELD) {
+ if (opcode == kFilterOpcode_SimpleField) {
msg_id = kSimpleField;
} else { // FILTER_OPCODE_NESTED_FIELD
// The next word in the bytecode contains the message index.
@@ -101,7 +119,7 @@ bool FilterBytecodeParser::LoadInternal(const uint8_t* filter_data,
i);
return false;
}
- msg_id = bytecode[++i];
+ msg_id = words[++i];
max_msg_index = std::max(max_msg_index, msg_id);
}
@@ -113,12 +131,12 @@ bool FilterBytecodeParser::LoadInternal(const uint8_t* filter_data,
// complexity to deal with rare cases like this.
add_range(field_id, field_id + 1, msg_id);
}
- } else if (opcode == ProtoFilter::FILTER_OPCODE_SIMPLE_FIELD_RANGE) {
+ } else if (opcode == kFilterOpcode_SimpleFieldRange) {
if (!has_next_word) {
PERFETTO_DLOG("bytecode error @ word %zu: unterminated range", i);
return false;
}
- const uint32_t range_len = bytecode[++i];
+ const uint32_t range_len = words[++i];
const uint32_t range_end = field_id + range_len; // STL-style, excl.
uint32_t id = field_id;
@@ -132,7 +150,7 @@ bool FilterBytecodeParser::LoadInternal(const uint8_t* filter_data,
PERFETTO_DCHECK(id >= kDirectlyIndexLimit || id == range_end);
if (id < range_end)
add_range(id, range_end, kSimpleField);
- } else if (opcode == ProtoFilter::FILTER_OPCODE_END_OF_MESSAGE) {
+ } else if (opcode == kFilterOpcode_EndOfMessage) {
// For each message append:
// 1. The "header" word telling how many directly indexed fields there
// are.
@@ -182,7 +200,8 @@ FilterBytecodeParser::QueryResult FilterBytecodeParser::Query(
// bytecode.
PERFETTO_DCHECK(start_offset < words_.size());
const uint32_t* word = &words_[start_offset];
- const uint32_t* const end = &words_[message_offset_[msg_index + 1]];
+ const uint32_t end_off = message_offset_[msg_index + 1];
+ const uint32_t* const end = words_.data() + end_off;
PERFETTO_DCHECK(end > word && end <= words_.data() + words_.size());
const uint32_t num_directly_indexed = *(word++);
PERFETTO_DCHECK(num_directly_indexed <= kDirectlyIndexLimit);
diff --git a/src/protozero/filtering/filter_bytecode_parser.h b/src/protozero/filtering/filter_bytecode_parser.h
index c71ec301f..b2378caca 100644
--- a/src/protozero/filtering/filter_bytecode_parser.h
+++ b/src/protozero/filtering/filter_bytecode_parser.h
@@ -24,10 +24,9 @@
namespace protozero {
-// Loads the proto-encoded bytecode (see proto_filter.proto) in memory and
-// allows fast lookups for tuples (msg_index, field_id) to tell if a given field
-// should be allowed or not and, in the case of nested fields, what is the next
-// message index to recurse into.
+// Loads the proto-encoded bytecode in memory and allows fast lookups for tuples
+// (msg_index, field_id) to tell if a given field should be allowed or not and,
+// in the case of nested fields, what is the next message index to recurse into.
// This class does two things:
// 1. Expands the array of varint from the proto into a vector<uint32_t>. This
// is to avoid performing varint decoding on every lookup, at the cost of
@@ -58,8 +57,8 @@ class FilterBytecodeParser {
bool simple_field() const { return nested_msg_index == kSimpleField; }
};
- // Loads a filter. The passed data must be proto-encoded bytes for a
- // perfetto.protos.ProtoFilter message.
+ // Loads a filter. The filter data consists of a sequence of varints which
+ // contains the filter opcodes and a final checksum.
bool Load(const void* filter_data, size_t len);
// Checks wheter a given field is allowed or not.
@@ -68,6 +67,7 @@ class FilterBytecodeParser {
QueryResult Query(uint32_t msg_index, uint32_t field_id);
void Reset();
+ void set_suppress_logs_for_fuzzer(bool x) { suppress_logs_for_fuzzer_ = x; }
private:
static constexpr uint32_t kDirectlyIndexLimit = 128;
@@ -115,6 +115,8 @@ class FilterBytecodeParser {
// Nth message start.
// message_offset_.size() - 2 == the max message id that can be parsed.
std::vector<uint32_t> message_offset_;
+
+ bool suppress_logs_for_fuzzer_ = false;
};
} // namespace protozero
diff --git a/src/protozero/filtering/filter_bytecode_parser_fuzzer.cc b/src/protozero/filtering/filter_bytecode_parser_fuzzer.cc
index 33dc5c911..578f6c919 100644
--- a/src/protozero/filtering/filter_bytecode_parser_fuzzer.cc
+++ b/src/protozero/filtering/filter_bytecode_parser_fuzzer.cc
@@ -18,14 +18,46 @@
#include <stdint.h>
#include <string.h>
+#include "perfetto/ext/base/hash.h"
+
+#include "perfetto/protozero/packed_repeated_fields.h"
#include "src/protozero/filtering/filter_bytecode_parser.h"
namespace protozero {
namespace {
+// This function gives a little help to the fuzzer. The bytecode is really a
+// sequence of varint-encoded uint32 words, with a FNV1a checksum at the end.
+// It's very unlikely that the fuzzer on its own can work out the checksum, so
+// most fuzzer inputs are doomed to fail the checksum verification.
+// This takes the fuzzer input and builds a more plausible bytecode.
+void LoadBytecodeWithChecksum(FilterBytecodeParser* parser,
+ const uint8_t* data,
+ size_t size) {
+ protozero::PackedVarInt words;
+ perfetto::base::Hash hasher;
+ for (size_t i = 0; i < size; i += sizeof(uint32_t)) {
+ uint32_t word = 0;
+ memcpy(&word, data, sizeof(uint32_t));
+ words.Append(word);
+ hasher.Update(word);
+ }
+ words.Append(static_cast<uint32_t>(hasher.digest()));
+ parser->Load(words.data(), words.size());
+}
+
int FuzzBytecodeParser(const uint8_t* data, size_t size) {
FilterBytecodeParser parser;
- parser.Load(data, size > 8 ? size - 8 : size);
+ parser.set_suppress_logs_for_fuzzer(true);
+
+ if (size > 4 && data[0] < 192) {
+ // 75% of the times use the LoadBytecodeWithChecksum() which helps the
+ // fuzzer passing the checksum verification.
+ LoadBytecodeWithChecksum(&parser, data + 1, size - 1);
+ } else {
+ // In the remaining 25%, pass completely arbitrary inputs.
+ parser.Load(data, size);
+ }
// Smoke testing with known problematic values
for (uint32_t msg_index = 0; msg_index < 3; ++msg_index) {
diff --git a/src/protozero/filtering/filter_bytecode_parser_unittest.cc b/src/protozero/filtering/filter_bytecode_parser_unittest.cc
index 28e4e4b28..5b67ded64 100644
--- a/src/protozero/filtering/filter_bytecode_parser_unittest.cc
+++ b/src/protozero/filtering/filter_bytecode_parser_unittest.cc
@@ -16,33 +16,32 @@
#include "test/gtest_and_gmock.h"
+#include "perfetto/ext/base/hash.h"
#include "perfetto/protozero/packed_repeated_fields.h"
#include "perfetto/protozero/scattered_heap_buffer.h"
+#include "src/protozero/filtering/filter_bytecode_common.h"
#include "src/protozero/filtering/filter_bytecode_parser.h"
-#include "protos/perfetto/config/proto_filter.pbzero.h"
-
namespace protozero {
namespace {
-using PF = perfetto::protos::pbzero::ProtoFilter;
bool LoadBytecode(FilterBytecodeParser* parser,
std::initializer_list<uint32_t> bytecode) {
+ perfetto::base::Hash hasher;
protozero::PackedVarInt words;
- for (uint32_t w : bytecode)
+ for (uint32_t w : bytecode) {
words.Append(w);
-
- HeapBuffered<PF> filter;
- filter->set_bytecode(words);
- auto filter_msg = filter.SerializeAsArray();
- return parser->Load(filter_msg.data(), filter_msg.size());
+ hasher.Update(w);
+ }
+ words.Append(static_cast<uint32_t>(hasher.digest()));
+ return parser->Load(words.data(), words.size());
}
TEST(FilterBytecodeParserTest, ParserSimpleFields) {
FilterBytecodeParser parser;
- EXPECT_TRUE(parser.Load(nullptr, 0));
+ EXPECT_FALSE(parser.Load(nullptr, 0));
EXPECT_FALSE(parser.Query(42, 42).allowed);
EXPECT_TRUE(LoadBytecode(&parser, {}));
@@ -54,12 +53,12 @@ TEST(FilterBytecodeParserTest, ParserSimpleFields) {
EXPECT_FALSE(parser.Query(42, 42).allowed);
// An invalid field_id (0) in bytecode should cause a parse failure.
- EXPECT_FALSE(LoadBytecode(&parser, {PF::FILTER_OPCODE_SIMPLE_FIELD | 0,
- PF::FILTER_OPCODE_END_OF_MESSAGE}));
+ EXPECT_FALSE(LoadBytecode(
+ &parser, {kFilterOpcode_SimpleField | 0, kFilterOpcode_EndOfMessage}));
// A valid bytecode that has only one field.
- EXPECT_TRUE(LoadBytecode(&parser, {PF::FILTER_OPCODE_SIMPLE_FIELD | (2u << 3),
- PF::FILTER_OPCODE_END_OF_MESSAGE}));
+ EXPECT_TRUE(LoadBytecode(&parser, {kFilterOpcode_SimpleField | (2u << 3),
+ kFilterOpcode_EndOfMessage}));
EXPECT_FALSE(parser.Query(0, 0).allowed);
EXPECT_FALSE(parser.Query(0, 1).allowed);
EXPECT_TRUE(parser.Query(0, 2).allowed);
@@ -71,12 +70,11 @@ TEST(FilterBytecodeParserTest, ParserSimpleFields) {
EXPECT_FALSE(parser.Query(1, 3).allowed);
// A valid bytecode that has few sparse fields < 128.
- EXPECT_TRUE(
- LoadBytecode(&parser, {PF::FILTER_OPCODE_SIMPLE_FIELD | (1u << 3),
- PF::FILTER_OPCODE_SIMPLE_FIELD | (7u << 3),
- PF::FILTER_OPCODE_SIMPLE_FIELD | (8u << 3),
- PF::FILTER_OPCODE_SIMPLE_FIELD | (127u << 3),
- PF::FILTER_OPCODE_END_OF_MESSAGE}));
+ EXPECT_TRUE(LoadBytecode(&parser, {kFilterOpcode_SimpleField | (1u << 3),
+ kFilterOpcode_SimpleField | (7u << 3),
+ kFilterOpcode_SimpleField | (8u << 3),
+ kFilterOpcode_SimpleField | (127u << 3),
+ kFilterOpcode_EndOfMessage}));
EXPECT_FALSE(parser.Query(0, 0).allowed);
EXPECT_TRUE(parser.Query(0, 1).allowed);
EXPECT_FALSE(parser.Query(0, 2).allowed);
@@ -90,11 +88,10 @@ TEST(FilterBytecodeParserTest, ParserSimpleFields) {
EXPECT_FALSE(parser.Query(0, 128).allowed);
// A valid bytecode that has only fields > 128.
- EXPECT_TRUE(
- LoadBytecode(&parser, {PF::FILTER_OPCODE_SIMPLE_FIELD | (1000u << 3),
- PF::FILTER_OPCODE_SIMPLE_FIELD | (1001u << 3),
- PF::FILTER_OPCODE_SIMPLE_FIELD | (2000u << 3),
- PF::FILTER_OPCODE_END_OF_MESSAGE}));
+ EXPECT_TRUE(LoadBytecode(&parser, {kFilterOpcode_SimpleField | (1000u << 3),
+ kFilterOpcode_SimpleField | (1001u << 3),
+ kFilterOpcode_SimpleField | (2000u << 3),
+ kFilterOpcode_EndOfMessage}));
for (uint32_t i = 0; i < 1000; ++i)
EXPECT_FALSE(parser.Query(0, i).allowed);
EXPECT_TRUE(parser.Query(0, 1000).allowed);
@@ -109,28 +106,28 @@ TEST(FilterBytecodeParserTest, ParserSimpleRanges) {
FilterBytecodeParser parser;
// Invalid, range length missing.
- EXPECT_FALSE(LoadBytecode(
- &parser, {
- PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (2u << 3),
- }));
+ EXPECT_FALSE(
+ LoadBytecode(&parser, {
+ kFilterOpcode_SimpleFieldRange | (2u << 3),
+ }));
// Borderline valid: range length = 0.
- EXPECT_TRUE(LoadBytecode(
- &parser, {PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (2u << 3), 0u,
- PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (127u << 3), 0u,
- PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (128u << 3), 0u,
- PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (128u << 3), 0u,
- PF::FILTER_OPCODE_END_OF_MESSAGE}));
+ EXPECT_TRUE(
+ LoadBytecode(&parser, {kFilterOpcode_SimpleFieldRange | (2u << 3), 0u,
+ kFilterOpcode_SimpleFieldRange | (127u << 3), 0u,
+ kFilterOpcode_SimpleFieldRange | (128u << 3), 0u,
+ kFilterOpcode_SimpleFieldRange | (128u << 3), 0u,
+ kFilterOpcode_EndOfMessage}));
for (uint32_t i = 0; i < 130; ++i)
EXPECT_FALSE(parser.Query(0, i).allowed) << i;
// A valid bytecode with two ranges [2,2], [10, 14].
EXPECT_TRUE(
- LoadBytecode(&parser, {PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (2u << 3),
+ LoadBytecode(&parser, {kFilterOpcode_SimpleFieldRange | (2u << 3),
1u, // length of the range,
- PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (10u << 3),
+ kFilterOpcode_SimpleFieldRange | (10u << 3),
5u, // length of the range,
- PF::FILTER_OPCODE_END_OF_MESSAGE}));
+ kFilterOpcode_EndOfMessage}));
EXPECT_FALSE(parser.Query(0, 0).allowed);
EXPECT_FALSE(parser.Query(0, 1).allowed);
EXPECT_TRUE(parser.Query(0, 2).allowed);
@@ -147,17 +144,17 @@ TEST(FilterBytecodeParserTest, ParserSimpleFieldsAndRanges) {
// Borderline valid: range length = 0.
EXPECT_TRUE(
- LoadBytecode(&parser, {PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (1u << 3),
+ LoadBytecode(&parser, {kFilterOpcode_SimpleFieldRange | (1u << 3),
2u, // [1,2]
- PF::FILTER_OPCODE_SIMPLE_FIELD | (4u << 3),
+ kFilterOpcode_SimpleField | (4u << 3),
- PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (126u << 3),
+ kFilterOpcode_SimpleFieldRange | (126u << 3),
4u, // [126, 129]
- PF::FILTER_OPCODE_SIMPLE_FIELD | (150u << 3),
+ kFilterOpcode_SimpleField | (150u << 3),
- PF::FILTER_OPCODE_END_OF_MESSAGE}));
+ kFilterOpcode_EndOfMessage}));
EXPECT_TRUE(parser.Query(0, 1).allowed);
EXPECT_TRUE(parser.Query(0, 2).allowed);
EXPECT_FALSE(parser.Query(0, 3).allowed);
@@ -175,42 +172,41 @@ TEST(FilterBytecodeParserTest, ParserNestedMessages) {
// Invalid because there are 1 messages in total, and message index 1 is
// out of range.
- EXPECT_FALSE(
- LoadBytecode(&parser, {PF::FILTER_OPCODE_NESTED_FIELD | (4u << 3),
- 1u, // message index
- PF::FILTER_OPCODE_END_OF_MESSAGE}));
+ EXPECT_FALSE(LoadBytecode(&parser, {kFilterOpcode_NestedField | (4u << 3),
+ 1u, // message index
+ kFilterOpcode_EndOfMessage}));
// A valid bytecode consisting of 4 message, with recursive / cylical
// dependencies between them.
EXPECT_TRUE(LoadBytecode(
&parser, {
// Message 0 (root).
- PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (1u << 3),
+ kFilterOpcode_SimpleFieldRange | (1u << 3),
2u, // [1,2]
- PF::FILTER_OPCODE_NESTED_FIELD | (4u << 3),
+ kFilterOpcode_NestedField | (4u << 3),
3u, // message index
- PF::FILTER_OPCODE_SIMPLE_FIELD | (10u << 3),
- PF::FILTER_OPCODE_NESTED_FIELD | (127u << 3),
+ kFilterOpcode_SimpleField | (10u << 3),
+ kFilterOpcode_NestedField | (127u << 3),
1u, // message index
- PF::FILTER_OPCODE_NESTED_FIELD | (128u << 3),
+ kFilterOpcode_NestedField | (128u << 3),
2u, // message index
- PF::FILTER_OPCODE_END_OF_MESSAGE,
+ kFilterOpcode_EndOfMessage,
// Message 1.
- PF::FILTER_OPCODE_NESTED_FIELD | (2u << 3),
+ kFilterOpcode_NestedField | (2u << 3),
1u, // message index (recurse onto itself),
- PF::FILTER_OPCODE_SIMPLE_FIELD | (11u << 3),
- PF::FILTER_OPCODE_END_OF_MESSAGE,
+ kFilterOpcode_SimpleField | (11u << 3),
+ kFilterOpcode_EndOfMessage,
// Message 2.
- PF::FILTER_OPCODE_NESTED_FIELD | (2u << 3),
+ kFilterOpcode_NestedField | (2u << 3),
3u, // message index.
- PF::FILTER_OPCODE_END_OF_MESSAGE,
+ kFilterOpcode_EndOfMessage,
// Message 3.
- PF::FILTER_OPCODE_NESTED_FIELD | (2u << 3),
+ kFilterOpcode_NestedField | (2u << 3),
2u, // message index (create a cycle, 2->3, 3->2).
- PF::FILTER_OPCODE_END_OF_MESSAGE,
+ kFilterOpcode_EndOfMessage,
}));
// Query message 0 fields.
diff --git a/src/protozero/filtering/filter_util.cc b/src/protozero/filtering/filter_util.cc
new file mode 100644
index 000000000..94d436e7a
--- /dev/null
+++ b/src/protozero/filtering/filter_util.cc
@@ -0,0 +1,323 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/protozero/filtering/filter_util.h"
+
+#include <algorithm>
+#include <map>
+#include <memory>
+#include <set>
+
+#include <google/protobuf/compiler/importer.h>
+
+#include "perfetto/base/build_config.h"
+#include "perfetto/ext/base/file_utils.h"
+#include "perfetto/ext/base/getopt.h"
+#include "perfetto/ext/base/string_utils.h"
+#include "perfetto/ext/base/version.h"
+#include "perfetto/protozero/proto_utils.h"
+#include "src/protozero/filtering/filter_bytecode_generator.h"
+
+namespace protozero {
+
+namespace {
+
+class MultiFileErrorCollectorImpl
+ : public google::protobuf::compiler::MultiFileErrorCollector {
+ public:
+ ~MultiFileErrorCollectorImpl() override;
+ void AddError(const std::string&, int, int, const std::string&) override;
+ void AddWarning(const std::string&, int, int, const std::string&) override;
+};
+
+MultiFileErrorCollectorImpl::~MultiFileErrorCollectorImpl() = default;
+
+void MultiFileErrorCollectorImpl::AddError(const std::string& filename,
+ int line,
+ int column,
+ const std::string& message) {
+ PERFETTO_ELOG("Error %s %d:%d: %s", filename.c_str(), line, column,
+ message.c_str());
+}
+
+void MultiFileErrorCollectorImpl::AddWarning(const std::string& filename,
+ int line,
+ int column,
+ const std::string& message) {
+ PERFETTO_ELOG("Warning %s %d:%d: %s", filename.c_str(), line, column,
+ message.c_str());
+}
+
+} // namespace
+
+FilterUtil::FilterUtil() = default;
+FilterUtil::~FilterUtil() = default;
+
+bool FilterUtil::LoadMessageDefinition(const std::string& proto_file,
+ const std::string& root_message,
+ const std::string& proto_dir_path) {
+ // The protobuf compiler doesn't like backslashes and prints an error like:
+ // Error C:\it7mjanpw3\perfetto-a16500 -1:0: Backslashes, consecutive slashes,
+ // ".", or ".." are not allowed in the virtual path.
+ // Given that C:\foo\bar is a legit path on windows, fix it at this level
+ // because the problem is really the protobuf compiler being too picky.
+ static auto normalize_for_win = [](const std::string& path) {
+#if PERFETTO_BUILDFLAG(PERFETTO_OS_WIN)
+ return perfetto::base::ReplaceAll(path, "\\", "/");
+#else
+ return path;
+#endif
+ };
+
+ google::protobuf::compiler::DiskSourceTree dst;
+#if PERFETTO_BUILDFLAG(PERFETTO_OS_WIN)
+ // If the path is absolute, maps "C:/" -> "C:/" (without hardcoding 'C').
+ if (proto_file.size() > 3 && proto_file[1] == ':') {
+ char win_drive[4];
+ sprintf(win_drive, "%c:/", proto_file[0]);
+ dst.MapPath(win_drive, win_drive);
+ }
+#endif
+ dst.MapPath("/", "/"); // We might still need this on Win under cygwin.
+ dst.MapPath("", normalize_for_win(proto_dir_path));
+ MultiFileErrorCollectorImpl mfe;
+ google::protobuf::compiler::Importer importer(&dst, &mfe);
+ const google::protobuf::FileDescriptor* root_file =
+ importer.Import(normalize_for_win(proto_file));
+ const google::protobuf::Descriptor* root_msg = nullptr;
+ if (!root_message.empty()) {
+ root_msg = importer.pool()->FindMessageTypeByName(root_message);
+ } else if (root_file->message_type_count() > 0) {
+ // The user didn't specfy the root type. Pick the first type in the file,
+ // most times it's the right guess.
+ root_msg = root_file->message_type(0);
+ if (root_msg)
+ PERFETTO_LOG(
+ "The guessed root message name is \"%s\". Pass -r com.MyName to "
+ "override",
+ root_msg->full_name().c_str());
+ }
+
+ if (!root_msg) {
+ PERFETTO_ELOG("Could not find the root message \"%s\" in %s",
+ root_message.c_str(), proto_file.c_str());
+ return false;
+ }
+
+ // |descriptors_by_full_name| is passed by argument rather than being a member
+ // field so that we don't risk leaving it out of sync (and depending on it in
+ // future without realizing) when performing the Dedupe() pass.
+ DescriptorsByNameMap descriptors_by_full_name;
+ ParseProtoDescriptor(root_msg, &descriptors_by_full_name);
+ return true;
+}
+
+// Generates a Message object for the given libprotobuf message descriptor.
+// Recurses as needed into nested fields.
+FilterUtil::Message* FilterUtil::ParseProtoDescriptor(
+ const google::protobuf::Descriptor* proto,
+ DescriptorsByNameMap* descriptors_by_full_name) {
+ auto descr_it = descriptors_by_full_name->find(proto->full_name());
+ if (descr_it != descriptors_by_full_name->end())
+ return descr_it->second;
+
+ descriptors_.emplace_back();
+ Message* msg = &descriptors_.back();
+ msg->full_name = proto->full_name();
+ (*descriptors_by_full_name)[msg->full_name] = msg;
+ for (int i = 0; i < proto->field_count(); ++i) {
+ const auto* proto_field = proto->field(i);
+ const uint32_t field_id = static_cast<uint32_t>(proto_field->number());
+ PERFETTO_CHECK(msg->fields.count(field_id) == 0);
+ auto& field = msg->fields[field_id];
+ field.name = proto_field->name();
+ field.type = proto_field->type_name();
+ if (proto_field->message_type()) {
+ msg->has_nested_fields = true;
+ // Recurse.
+ field.nested_type = ParseProtoDescriptor(proto_field->message_type(),
+ descriptors_by_full_name);
+ }
+ }
+ return msg;
+}
+
+void FilterUtil::Dedupe() {
+ std::map<std::string /*identity*/, Message*> index;
+
+ std::map<Message*, Message*> dupe_graph; // K,V: K shall be duped against V.
+
+ // As a first pass, generate an |identity| string for each leaf message. The
+ // identity is simply the comma-separated stringification of its field ids.
+ // If another message with the same identity exists, add an edge to the graph.
+ const size_t initial_count = descriptors_.size();
+ size_t field_count = 0;
+ for (auto& descr : descriptors_) {
+ if (descr.has_nested_fields)
+ continue; // Dedupe only leaf messages without nested fields.
+ std::string identity;
+ for (const auto& id_and_field : descr.fields)
+ identity.append(std::to_string(id_and_field.first) + ",");
+ auto it_and_inserted = index.emplace(identity, &descr);
+ if (!it_and_inserted.second) {
+ // insertion failed, a dupe exists already.
+ Message* dupe_against = it_and_inserted.first->second;
+ dupe_graph.emplace(&descr, dupe_against);
+ }
+ }
+
+ // Now apply de-duplications by re-directing the nested_type pointer to the
+ // equivalent descriptors that have the same set of allowed field ids.
+ std::set<Message*> referenced_descriptors;
+ referenced_descriptors.emplace(&descriptors_.front()); // The root.
+ for (auto& descr : descriptors_) {
+ for (auto& id_and_field : descr.fields) {
+ Message* target = id_and_field.second.nested_type;
+ if (!target)
+ continue; // Only try to dedupe nested types.
+ auto it = dupe_graph.find(target);
+ if (it == dupe_graph.end()) {
+ referenced_descriptors.emplace(target);
+ continue;
+ }
+ ++field_count;
+ // Replace with the dupe.
+ id_and_field.second.nested_type = it->second;
+ } // for (nested_fields).
+ } // for (descriptors_).
+
+ // Remove unreferenced descriptors. We should much rather crash in the case of
+ // a logic bug rathern than trying to use them but don't emit them.
+ size_t removed_count = 0;
+ for (auto it = descriptors_.begin(); it != descriptors_.end();) {
+ if (referenced_descriptors.count(&*it)) {
+ ++it;
+ } else {
+ ++removed_count;
+ it = descriptors_.erase(it);
+ }
+ }
+ PERFETTO_LOG(
+ "Deduplication removed %zu duped descriptors out of %zu descriptors from "
+ "%zu fields",
+ removed_count, initial_count, field_count);
+}
+
+// Prints the list of messages and fields in a diff-friendly text format.
+void FilterUtil::PrintAsText() {
+ using perfetto::base::StripPrefix;
+ const std::string& root_name = descriptors_.front().full_name;
+ std::string root_prefix = root_name.substr(0, root_name.rfind('.'));
+ if (!root_prefix.empty())
+ root_prefix.append(".");
+
+ for (const auto& descr : descriptors_) {
+ for (const auto& id_and_field : descr.fields) {
+ const uint32_t field_id = id_and_field.first;
+ const auto& field = id_and_field.second;
+ const Message* nested_type = id_and_field.second.nested_type;
+ auto stripped_name = StripPrefix(descr.full_name, root_prefix);
+ std::string stripped_nested =
+ nested_type ? StripPrefix(nested_type->full_name, root_prefix) : "";
+ printf("%-60s %3u %-8s %-32s %s\n", stripped_name.c_str(), field_id,
+ field.type.c_str(), field.name.c_str(), stripped_nested.c_str());
+ }
+ }
+}
+
+std::string FilterUtil::GenerateFilterBytecode() {
+ protozero::FilterBytecodeGenerator bytecode_gen;
+
+ // Assign indexes to descriptors, simply by counting them in order;
+ std::map<Message*, uint32_t> descr_to_idx;
+ for (auto& descr : descriptors_)
+ descr_to_idx[&descr] = static_cast<uint32_t>(descr_to_idx.size());
+
+ for (auto& descr : descriptors_) {
+ for (auto it = descr.fields.begin(); it != descr.fields.end();) {
+ uint32_t field_id = it->first;
+ const Message::Field& field = it->second;
+ if (field.nested_type) {
+ // Append the index of the target submessage.
+ PERFETTO_CHECK(descr_to_idx.count(field.nested_type));
+ uint32_t nested_msg_index = descr_to_idx[field.nested_type];
+ bytecode_gen.AddNestedField(field_id, nested_msg_index);
+ ++it;
+ continue;
+ }
+ // Simple field. Lookahead to see if we have a range of contiguous simple
+ // fields.
+ for (uint32_t range_len = 1;; ++range_len) {
+ ++it;
+ if (it != descr.fields.end() && it->first == field_id + range_len &&
+ it->second.is_simple()) {
+ continue;
+ }
+ // At this point it points to either the end() of the vector or a
+ // non-contiguous or non-simple field (which will be picked up by the
+ // next iteration).
+ if (range_len == 1) {
+ bytecode_gen.AddSimpleField(field_id);
+ } else {
+ bytecode_gen.AddSimpleFieldRange(field_id, range_len);
+ }
+ break;
+ } // for (range_len)
+ } // for (descr.fields)
+ bytecode_gen.EndMessage();
+ } // for (descriptors)
+ return bytecode_gen.Serialize();
+}
+
+std::string FilterUtil::LookupField(const std::string& varint_encoded_path) {
+ const uint8_t* ptr =
+ reinterpret_cast<const uint8_t*>(varint_encoded_path.data());
+ const uint8_t* const end = ptr + varint_encoded_path.size();
+
+ std::vector<uint32_t> fields;
+ while (ptr < end) {
+ uint64_t varint;
+ const uint8_t* next = proto_utils::ParseVarInt(ptr, end, &varint);
+ PERFETTO_CHECK(next != ptr);
+ fields.emplace_back(static_cast<uint32_t>(varint));
+ ptr = next;
+ }
+ return LookupField(fields.data(), fields.size());
+}
+
+std::string FilterUtil::LookupField(const uint32_t* field_ids,
+ size_t num_fields) {
+ const Message* msg = descriptors_.empty() ? nullptr : &descriptors_.front();
+ std::string res;
+ for (size_t i = 0; i < num_fields; ++i) {
+ const uint32_t field_id = field_ids[i];
+ const Message::Field* field = nullptr;
+ if (msg) {
+ auto it = msg->fields.find(field_id);
+ field = it == msg->fields.end() ? nullptr : &it->second;
+ }
+ res.append(".");
+ if (field) {
+ res.append(field->name);
+ msg = field->nested_type;
+ } else {
+ res.append(std::to_string(field_id));
+ }
+ }
+ return res;
+}
+
+} // namespace protozero
diff --git a/src/protozero/filtering/filter_util.h b/src/protozero/filtering/filter_util.h
new file mode 100644
index 000000000..3cc7655c8
--- /dev/null
+++ b/src/protozero/filtering/filter_util.h
@@ -0,0 +1,107 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SRC_PROTOZERO_FILTERING_FILTER_UTIL_H_
+#define SRC_PROTOZERO_FILTERING_FILTER_UTIL_H_
+
+#include <stdint.h>
+
+#include <list>
+#include <map>
+#include <string>
+
+namespace google {
+namespace protobuf {
+class Descriptor;
+}
+} // namespace google
+
+namespace protozero {
+
+// Parses a .proto message definition, recursing into its sub-messages, and
+// builds up a set of Messages and Field definitions.
+// Depends on libprotobuf-full and should be used only in host tools.
+// See the //tools/proto_filter for an executable that wraps this class with
+// a cmdline interface.
+class FilterUtil {
+ public:
+ FilterUtil();
+ ~FilterUtil();
+
+ // Loads a message schema from a .proto file, recursing into nested types.
+ // Args:
+ // proto_file: path to the .proto file.
+ // root_message: fully qualified message name (e.g., perfetto.protos.Trace).
+ // If empty, the first message in the file will be used.
+ // proto_dir_path: the root for .proto includes. If empty uses CWD.
+ bool LoadMessageDefinition(const std::string& proto_file,
+ const std::string& root_message,
+ const std::string& proto_dir_path);
+
+ // Deduplicates leaf messages having the same sets of field ids.
+ // It changes the internal state and affects the behavior of next calls to
+ // GenerateFilterBytecode() and PrintAsText().
+ void Dedupe();
+
+ // Generates the filter bytecode for the root message previously loaded by
+ // LoadMessageDefinition() using FilterBytecodeGenerator.
+ // The returned string is a binary-encoded proto message of type
+ // perfetto.protos.ProtoFilter (see proto_filter.proto).
+ std::string GenerateFilterBytecode();
+
+ // Prints the list of messages and fields onto stdout in a diff-friendly text
+ // format. Example:
+ // PowerRails 2 message energy_data PowerRails.EnergyData
+ // PowerRails.RailDescriptor 1 uint32 index
+ void PrintAsText();
+
+ // Resolves an array of field ids into a dot-concatenated field names.
+ // E.g., [2,5,1] -> ".trace.packet.timestamp".
+ std::string LookupField(const uint32_t* field_ids, size_t num_fields);
+
+ // Like the above but the array of field is passed as a buffer containing
+ // varints, e.g. "\x02\x05\0x01".
+ std::string LookupField(const std::string& varint_encoded_path);
+
+ private:
+ struct Message {
+ struct Field {
+ std::string name;
+ std::string type; // "uint32", "string", "message"
+ // Only when type == "message". Note that when using Dedupe() this can
+ // be aliased against a different submessage which happens to have the
+ // same set of field ids.
+ Message* nested_type = nullptr;
+ bool is_simple() const { return nested_type == nullptr; }
+ };
+ std::string full_name; // e.g., "perfetto.protos.Foo.Bar";
+ std::map<uint32_t /*field_id*/, Field> fields;
+
+ // True if at least one field has a non-null |nestd_type|.
+ bool has_nested_fields = false;
+ };
+
+ using DescriptorsByNameMap = std::map<std::string, Message*>;
+ Message* ParseProtoDescriptor(const google::protobuf::Descriptor*,
+ DescriptorsByNameMap*);
+
+ // list<> because pointers need to be stable.
+ std::list<Message> descriptors_;
+};
+
+} // namespace protozero
+
+#endif // SRC_PROTOZERO_FILTERING_FILTER_UTIL_H_
diff --git a/src/protozero/filtering/filter_util_unittest.cc b/src/protozero/filtering/filter_util_unittest.cc
new file mode 100644
index 000000000..9c7cf0d6c
--- /dev/null
+++ b/src/protozero/filtering/filter_util_unittest.cc
@@ -0,0 +1,182 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "test/gtest_and_gmock.h"
+
+#include "perfetto/ext/base/file_utils.h"
+#include "perfetto/ext/base/temp_file.h"
+#include "src/protozero/filtering/filter_bytecode_parser.h"
+#include "src/protozero/filtering/filter_util.h"
+
+namespace protozero {
+
+namespace {
+
+perfetto::base::TempFile MkTemp(const char* str) {
+ auto tmp = perfetto::base::TempFile::Create();
+ perfetto::base::WriteAll(*tmp, str, strlen(str));
+ perfetto::base::FlushFile(*tmp);
+ return tmp;
+}
+
+TEST(SchemaParserTest, SchemaToBytecode_Simple) {
+ auto schema = MkTemp(R"(
+ syntax = "proto2";
+ message Root {
+ optional int32 i32 = 13;
+ optional fixed64 f64 = 5;
+ optional string str = 71;
+ }
+ )");
+ FilterUtil filter;
+ ASSERT_TRUE(filter.LoadMessageDefinition(schema.path(), "Root", ""));
+ std::string bytecode = filter.GenerateFilterBytecode();
+ FilterBytecodeParser fbp;
+ ASSERT_TRUE(fbp.Load(bytecode.data(), bytecode.size()));
+ EXPECT_TRUE(fbp.Query(0, 13).allowed);
+ EXPECT_TRUE(fbp.Query(0, 13).simple_field());
+ EXPECT_TRUE(fbp.Query(0, 5).allowed);
+ EXPECT_TRUE(fbp.Query(0, 5).simple_field());
+ EXPECT_TRUE(fbp.Query(0, 71).allowed);
+ EXPECT_TRUE(fbp.Query(0, 71).simple_field());
+ EXPECT_FALSE(fbp.Query(0, 1).allowed);
+ EXPECT_FALSE(fbp.Query(0, 12).allowed);
+ EXPECT_FALSE(fbp.Query(0, 70).allowed);
+}
+
+TEST(SchemaParserTest, SchemaToBytecode_Nested) {
+ auto schema = MkTemp(R"(
+ syntax = "proto2";
+ message Root {
+ message Child {
+ repeated fixed64 f64 = 3;
+ optional Child recurse = 4;
+ }
+ oneof xxx { int32 i32 = 1; }
+ optional Child chld = 2;
+ }
+ )");
+ FilterUtil filter;
+ ASSERT_TRUE(filter.LoadMessageDefinition(schema.path(), "", ""));
+ std::string bytecode = filter.GenerateFilterBytecode();
+ FilterBytecodeParser fbp;
+ ASSERT_TRUE(fbp.Load(bytecode.data(), bytecode.size()));
+ EXPECT_TRUE(fbp.Query(0, 1).allowed);
+ EXPECT_TRUE(fbp.Query(0, 1).simple_field());
+ EXPECT_TRUE(fbp.Query(0, 2).allowed);
+ EXPECT_FALSE(fbp.Query(0, 2).simple_field());
+ // False as those fields exist only in Child, not in the root (0).
+ EXPECT_FALSE(fbp.Query(0, 3).allowed);
+ EXPECT_FALSE(fbp.Query(0, 4).allowed);
+
+ EXPECT_TRUE(fbp.Query(1, 3).allowed);
+ EXPECT_TRUE(fbp.Query(1, 3).simple_field());
+ EXPECT_TRUE(fbp.Query(1, 4).allowed);
+ EXPECT_FALSE(fbp.Query(1, 4).simple_field());
+ EXPECT_EQ(fbp.Query(1, 4).nested_msg_index, 1u); // Self
+}
+
+TEST(SchemaParserTest, SchemaToBytecode_Dedupe) {
+ auto schema = MkTemp(R"(
+ syntax = "proto2";
+ message Root {
+ message Nested {
+ message Child1 {
+ optional int32 f1 = 3;
+ optional int64 f2 = 4;
+ }
+ message Child2 {
+ optional string f1 = 3;
+ optional bytes f2 = 4;
+ }
+ message ChildNonDedupe {
+ optional string f1 = 3;
+ optional bytes f2 = 4;
+ optional int32 extra = 1;
+ }
+ optional Child1 chld1 = 1;
+ optional Child2 chld2 = 2;
+ optional ChildNonDedupe chld3 = 3;
+ }
+ repeated Nested nested = 1;
+ }
+ )");
+ FilterUtil filter;
+ ASSERT_TRUE(filter.LoadMessageDefinition(schema.path(), "Root", ""));
+ filter.Dedupe();
+ std::string bytecode = filter.GenerateFilterBytecode();
+ FilterBytecodeParser fbp;
+ ASSERT_TRUE(fbp.Load(bytecode.data(), bytecode.size()));
+
+ // 0: Root
+ EXPECT_TRUE(fbp.Query(0, 1).allowed);
+ EXPECT_FALSE(fbp.Query(0, 1).simple_field());
+
+ // 1: Nested
+ EXPECT_TRUE(fbp.Query(1, 1).allowed);
+ EXPECT_FALSE(fbp.Query(1, 1).simple_field());
+ EXPECT_TRUE(fbp.Query(1, 2).allowed);
+ EXPECT_FALSE(fbp.Query(1, 2).simple_field());
+ EXPECT_TRUE(fbp.Query(1, 3).allowed);
+ EXPECT_FALSE(fbp.Query(1, 3).simple_field());
+
+ // Check deduping.
+ // Fields chld1 and chld2 should point to the same sub-filter because they
+ // have the same field ids.
+ EXPECT_EQ(fbp.Query(1, 1).nested_msg_index, fbp.Query(1, 2).nested_msg_index);
+
+ // Field chld3 should point to a different one because it has an extra field.
+ EXPECT_NE(fbp.Query(1, 1).nested_msg_index, fbp.Query(1, 3).nested_msg_index);
+}
+
+TEST(SchemaParserTest, FieldLookup) {
+ auto schema = MkTemp(R"(
+ syntax = "proto2";
+ message Root {
+ message Nested {
+ message Child1 {
+ optional int32 f1 = 3;
+ optional int64 f2 = 4;
+ repeated Child2 c2 = 5;
+ }
+ message Child2 {
+ optional string f3 = 6;
+ optional bytes f4 = 7;
+ repeated Child1 c1 = 8;
+ }
+ optional Child1 x1 = 1;
+ optional Child2 x2 = 2;
+ }
+ repeated Nested n = 1;
+ }
+ )");
+
+ FilterUtil filter;
+ ASSERT_TRUE(filter.LoadMessageDefinition(schema.path(), "Root", ""));
+ std::vector<uint32_t> fld;
+
+ fld = {1, 1, 3};
+ ASSERT_EQ(filter.LookupField(fld.data(), fld.size()), ".n.x1.f1");
+
+ fld = {1, 2, 7};
+ ASSERT_EQ(filter.LookupField(fld.data(), fld.size()), ".n.x2.f4");
+
+ fld = {1, 2, 8, 5, 8, 5, 7};
+ ASSERT_EQ(filter.LookupField(fld.data(), fld.size()), ".n.x2.c1.c2.c1.c2.f4");
+}
+
+} // namespace
+} // namespace protozero
diff --git a/src/protozero/filtering/message_filter.cc b/src/protozero/filtering/message_filter.cc
new file mode 100644
index 000000000..a971dfab3
--- /dev/null
+++ b/src/protozero/filtering/message_filter.cc
@@ -0,0 +1,310 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/protozero/filtering/message_filter.h"
+
+#include "perfetto/base/logging.h"
+#include "perfetto/protozero/proto_utils.h"
+
+namespace protozero {
+
+namespace {
+
+// Inline helpers to append proto fields in output. They are the equivalent of
+// the protozero::Message::AppendXXX() fields but don't require building and
+// maintaining a full protozero::Message object or dealing with scattered
+// output slices.
+// All these functions assume there is enough space in the output buffer, which
+// should be always the case assuming that we don't end up generating more
+// output than input.
+
+inline void AppendVarInt(uint32_t field_id, uint64_t value, uint8_t** out) {
+ *out = proto_utils::WriteVarInt(proto_utils::MakeTagVarInt(field_id), *out);
+ *out = proto_utils::WriteVarInt(value, *out);
+}
+
+// For fixed32 / fixed64.
+template <typename INT_T /* uint32_t | uint64_t*/>
+inline void AppendFixed(uint32_t field_id, INT_T value, uint8_t** out) {
+ *out = proto_utils::WriteVarInt(proto_utils::MakeTagFixed<INT_T>(field_id),
+ *out);
+ memcpy(*out, &value, sizeof(value));
+ *out += sizeof(value);
+}
+
+// For length-delimited (string, bytes) fields. Note: this function appends only
+// the proto preamble and the varint field that states the length of the payload
+// not the payload itself.
+// In the case of submessages, the caller needs to re-write the length at the
+// end in the in the returned memory area.
+// The problem here is that, because of filtering, the length of a submessage
+// might be < original length (the original length is still an upper-bound).
+// Returns a pair with: (1) the pointer where the final length should be written
+// into, (2) the length of the size field.
+// The caller must write a redundant varint to match the original size (i.e.
+// needs to use WriteRedundantVarInt()).
+inline std::pair<uint8_t*, uint32_t> AppendLenDelim(uint32_t field_id,
+ uint32_t len,
+ uint8_t** out) {
+ *out = proto_utils::WriteVarInt(proto_utils::MakeTagLengthDelimited(field_id),
+ *out);
+ uint8_t* size_field_start = *out;
+ *out = proto_utils::WriteVarInt(len, *out);
+ const size_t size_field_len = static_cast<size_t>(*out - size_field_start);
+ return std::make_pair(size_field_start, size_field_len);
+}
+} // namespace
+
+MessageFilter::MessageFilter() {
+ // Push a state on the stack for the implicit root message.
+ stack_.emplace_back();
+}
+
+MessageFilter::~MessageFilter() = default;
+
+bool MessageFilter::LoadFilterBytecode(const void* filter_data, size_t len) {
+ return filter_.Load(filter_data, len);
+}
+
+bool MessageFilter::SetFilterRoot(const uint32_t* field_ids,
+ size_t num_fields) {
+ uint32_t root_msg_idx = 0;
+ for (const uint32_t* it = field_ids; it < field_ids + num_fields; ++it) {
+ uint32_t field_id = *it;
+ auto res = filter_.Query(root_msg_idx, field_id);
+ if (!res.allowed || res.simple_field())
+ return false;
+ root_msg_idx = res.nested_msg_index;
+ }
+ root_msg_index_ = root_msg_idx;
+ return true;
+}
+
+MessageFilter::FilteredMessage MessageFilter::FilterMessageFragments(
+ const InputSlice* slices,
+ size_t num_slices) {
+ // First compute the upper bound for the output. The filtered message cannot
+ // be > the original message.
+ uint32_t total_len = 0;
+ for (size_t i = 0; i < num_slices; ++i)
+ total_len += slices[i].len;
+ out_buf_.reset(new uint8_t[total_len]);
+ out_ = out_buf_.get();
+ out_end_ = out_ + total_len;
+
+ // Reset the parser state.
+ tokenizer_ = MessageTokenizer();
+ error_ = false;
+ stack_.clear();
+ stack_.resize(2);
+ // stack_[0] is a sentinel and should never be hit in nominal cases. If we
+ // end up there we will just keep consuming the input stream and detecting
+ // at the end, without hurting the fastpath.
+ stack_[0].in_bytes_limit = UINT32_MAX;
+ stack_[0].eat_next_bytes = UINT32_MAX;
+ // stack_[1] is the actual root message.
+ stack_[1].in_bytes_limit = total_len;
+ stack_[1].msg_index = root_msg_index_;
+
+ // Process the input data and write the output.
+ for (size_t slice_idx = 0; slice_idx < num_slices; ++slice_idx) {
+ const InputSlice& slice = slices[slice_idx];
+ const uint8_t* data = static_cast<const uint8_t*>(slice.data);
+ for (size_t i = 0; i < slice.len; ++i)
+ FilterOneByte(data[i]);
+ }
+
+ // Construct the output object.
+ PERFETTO_CHECK(out_ >= out_buf_.get() && out_ <= out_end_);
+ auto used_size = static_cast<size_t>(out_ - out_buf_.get());
+ FilteredMessage res{std::move(out_buf_), used_size};
+ res.error = error_;
+ if (stack_.size() != 1 || !tokenizer_.idle() ||
+ stack_[0].in_bytes != total_len) {
+ res.error = true;
+ }
+ return res;
+}
+
+void MessageFilter::FilterOneByte(uint8_t octet) {
+ PERFETTO_DCHECK(!stack_.empty());
+
+ auto* state = &stack_.back();
+ StackState next_state{};
+ bool push_next_state = false;
+
+ if (state->eat_next_bytes > 0) {
+ // This is the case where the previous tokenizer_.Push() call returned a
+ // length delimited message which is NOT a submessage (a string or a bytes
+ // field). We just want to consume it, and pass it through in output
+ // if the field was allowed.
+ --state->eat_next_bytes;
+ if (state->passthrough_eaten_bytes)
+ *(out_++) = octet;
+ } else {
+ MessageTokenizer::Token token = tokenizer_.Push(octet);
+ // |token| will not be valid() in most cases and this is WAI. When pushing
+ // a varint field, only the last byte yields a token, all the other bytes
+ // return an invalid token, they just update the internal tokenizer state.
+ if (token.valid()) {
+ auto filter = filter_.Query(state->msg_index, token.field_id);
+ switch (token.type) {
+ case proto_utils::ProtoWireType::kVarInt:
+ if (filter.allowed && filter.simple_field())
+ AppendVarInt(token.field_id, token.value, &out_);
+ break;
+ case proto_utils::ProtoWireType::kFixed32:
+ if (filter.allowed && filter.simple_field())
+ AppendFixed(token.field_id, static_cast<uint32_t>(token.value),
+ &out_);
+ break;
+ case proto_utils::ProtoWireType::kFixed64:
+ if (filter.allowed && filter.simple_field())
+ AppendFixed(token.field_id, static_cast<uint64_t>(token.value),
+ &out_);
+ break;
+ case proto_utils::ProtoWireType::kLengthDelimited:
+ // Here we have two cases:
+ // A. A simple string/bytes field: we just want to consume the next
+ // bytes (the string payload), optionally passing them through in
+ // output if the field is allowed.
+ // B. This is a nested submessage. In this case we want to recurse and
+ // push a new state on the stack.
+ // Note that we can't tell the difference between a
+ // "non-allowed string" and a "non-allowed submessage". But it doesn't
+ // matter because in both cases we just want to skip the next N bytes.
+ const auto submessage_len = static_cast<uint32_t>(token.value);
+ auto in_bytes_left = state->in_bytes_limit - state->in_bytes - 1;
+ if (PERFETTO_UNLIKELY(submessage_len > in_bytes_left)) {
+ // This is a malicious / malformed string/bytes/submessage that
+ // claims to be larger than the outer message that contains it.
+ return SetUnrecoverableErrorState();
+ }
+
+ if (filter.allowed && !filter.simple_field() && submessage_len > 0) {
+ // submessage_len == 0 is the edge case of a message with a 0-len
+ // (but present) submessage. In this case, if allowed, we don't want
+ // to push any further state (doing so would desync the FSM) but we
+ // still want to emit it.
+ // At this point |submessage_len| is only an upper bound. The
+ // final message written in output can be <= the one in input,
+ // only some of its fields might be allowed (also remember that
+ // this class implicitly removes redundancy varint encoding of
+ // len-delimited field lengths). The final length varint (the
+ // return value of AppendLenDelim()) will be filled when popping
+ // from |stack_|.
+ auto size_field =
+ AppendLenDelim(token.field_id, submessage_len, &out_);
+ push_next_state = true;
+ next_state.field_id = token.field_id;
+ next_state.msg_index = filter.nested_msg_index;
+ next_state.in_bytes_limit = submessage_len;
+ next_state.size_field = size_field.first;
+ next_state.size_field_len = size_field.second;
+ next_state.out_bytes_written_at_start = out_written();
+ } else {
+ // A string or bytes field, or a 0 length submessage.
+ state->eat_next_bytes = submessage_len;
+ state->passthrough_eaten_bytes = filter.allowed;
+ if (filter.allowed)
+ AppendLenDelim(token.field_id, submessage_len, &out_);
+ }
+ break;
+ } // switch(type)
+
+ if (PERFETTO_UNLIKELY(track_field_usage_)) {
+ IncrementCurrentFieldUsage(token.field_id, filter.allowed);
+ }
+ } // if (token.valid)
+ } // if (eat_next_bytes == 0)
+
+ ++state->in_bytes;
+ while (state->in_bytes >= state->in_bytes_limit) {
+ PERFETTO_DCHECK(state->in_bytes == state->in_bytes_limit);
+ push_next_state = false;
+
+ // We can't possibly write more than we read.
+ const uint32_t msg_bytes_written = static_cast<uint32_t>(
+ out_written() - state->out_bytes_written_at_start);
+ PERFETTO_DCHECK(msg_bytes_written <= state->in_bytes_limit);
+
+ // Backfill the length field of the
+ proto_utils::WriteRedundantVarInt(msg_bytes_written, state->size_field,
+ state->size_field_len);
+
+ const uint32_t in_bytes_processes_for_last_msg = state->in_bytes;
+ stack_.pop_back();
+ PERFETTO_CHECK(!stack_.empty());
+ state = &stack_.back();
+ state->in_bytes += in_bytes_processes_for_last_msg;
+ if (PERFETTO_UNLIKELY(!tokenizer_.idle())) {
+ // If we hit this case, it means that we got to the end of a submessage
+ // while decoding a field. We can't recover from this and we don't want to
+ // propagate a broken sub-message.
+ return SetUnrecoverableErrorState();
+ }
+ }
+
+ if (push_next_state) {
+ PERFETTO_DCHECK(tokenizer_.idle());
+ stack_.emplace_back(std::move(next_state));
+ state = &stack_.back();
+ }
+}
+
+void MessageFilter::SetUnrecoverableErrorState() {
+ error_ = true;
+ stack_.clear();
+ stack_.resize(1);
+ auto& state = stack_[0];
+ state.eat_next_bytes = UINT32_MAX;
+ state.in_bytes_limit = UINT32_MAX;
+ state.passthrough_eaten_bytes = false;
+ out_ = out_buf_.get(); // Reset the write pointer.
+}
+
+void MessageFilter::IncrementCurrentFieldUsage(uint32_t field_id,
+ bool allowed) {
+ // Slowpath. Used mainly in offline tools and tests to workout used fields in
+ // a proto.
+ PERFETTO_DCHECK(track_field_usage_);
+
+ // Field path contains a concatenation of varints, one for each nesting level.
+ // e.g. y in message Root { Sub x = 2; }; message Sub { SubSub y = 7; }
+ // is encoded as [varint(2) + varint(7)].
+ // We use varint to take the most out of SSO (small string opt). In most cases
+ // the path will fit in the on-stack 22 bytes, requiring no heap.
+ std::string field_path;
+
+ auto append_field_id = [&field_path](uint32_t id) {
+ uint8_t buf[10];
+ uint8_t* end = proto_utils::WriteVarInt(id, buf);
+ field_path.append(reinterpret_cast<char*>(buf),
+ static_cast<size_t>(end - buf));
+ };
+
+ // Append all the ancestors IDs from the state stack.
+ // The first entry of the stack has always ID 0 and we skip it (we don't know
+ // the ID of the root message itself).
+ PERFETTO_DCHECK(stack_.size() >= 2 && stack_[1].field_id == 0);
+ for (size_t i = 2; i < stack_.size(); ++i)
+ append_field_id(stack_[i].field_id);
+ // Append the id of the field in the current message.
+ append_field_id(field_id);
+ field_usage_[field_path] += allowed ? 1 : -1;
+}
+
+} // namespace protozero
diff --git a/src/protozero/filtering/message_filter.h b/src/protozero/filtering/message_filter.h
new file mode 100644
index 000000000..37dc9b520
--- /dev/null
+++ b/src/protozero/filtering/message_filter.h
@@ -0,0 +1,208 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SRC_PROTOZERO_FILTERING_MESSAGE_FILTER_H_
+#define SRC_PROTOZERO_FILTERING_MESSAGE_FILTER_H_
+
+#include <stdint.h>
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+#include "src/protozero/filtering/filter_bytecode_parser.h"
+#include "src/protozero/filtering/message_tokenizer.h"
+
+namespace protozero {
+
+// A class to filter binary-encoded proto messages using an allow-list of field
+// ids, also known as "filter bytecode". The filter determines which fields are
+// allowed to be passed through in output and strips all the other fields.
+// See go/trace-filtering for full design.
+// This class takes in input:
+// 1) The filter bytecode, loaded once via the LoadFilterBytecode() method.
+// 2) A proto-encoded binary message. The message doesn't have to be contiguous,
+// it can be passed as an array of arbitrarily chunked fragments.
+// The FilterMessage*() method returns in output a proto message, stripping out
+// all unknown fields. If the input is malformed (e.g., unknown proto field wire
+// types, lengths out of bound) the whole filtering failed and the |error| flag
+// of the FilteredMessage object is set to true.
+// The filtering operation is based on rewriting a copy of the message into a
+// self-allocated buffer, which is then returned in the output. The input buffer
+// is NOT altered.
+// Note also that the process of rewriting the protos gets rid of most redundant
+// varint encoding (if present). So even if all fields are allow-listed, the
+// output might NOT be bitwise identical to the input (but it will be
+// semantically equivalent).
+// Furthermore the enable_field_usage_tracking() method allows to keep track of
+// a histogram of allowed / denied fields. It slows down filtering and is
+// intended only on host tools.
+class MessageFilter {
+ public:
+ MessageFilter();
+ ~MessageFilter();
+
+ struct InputSlice {
+ const void* data;
+ size_t len;
+ };
+
+ struct FilteredMessage {
+ FilteredMessage(std::unique_ptr<uint8_t[]> d, size_t s)
+ : data(std::move(d)), size(s) {}
+ std::unique_ptr<uint8_t[]> data;
+ size_t size; // The used bytes in |data|. This is <= sizeof(data).
+ bool error = false;
+ };
+
+ // Loads the filter bytecode that will be used to filter any subsequent
+ // message. Must be called before the first call to FilterMessage*().
+ // |filter_data| must point to a byte buffer for a proto-encoded ProtoFilter
+ // message (see proto_filter.proto).
+ bool LoadFilterBytecode(const void* filter_data, size_t len);
+
+ // This affects the filter starting point of the subsequent FilterMessage*()
+ // calls. By default the filtering process starts from the message @ index 0,
+ // the root message passed to proto_filter when generating the bytecode
+ // (in typical tracing use-cases, this is perfetto.protos.Trace). However, the
+ // caller (TracingServiceImpl) might want to filter packets from the 2nd level
+ // (perfetto.protos.TracePacket) because the root level is pre-pended after
+ // the fact. This call allows to change the root message for the filter.
+ // The argument |field_ids| is an array of proto field ids and determines the
+ // path to the new root. For instance, in the case of [1,2,3] SetFilterRoot
+ // will identify the sub-message for the field "root.1.2.3" and use that.
+ // In order for this to succeed all the fields in the path must be allowed
+ // in the filter and must be a nested message type.
+ bool SetFilterRoot(const uint32_t* field_ids, size_t num_fields);
+
+ // Takes an input message, fragmented in arbitrary slices, and returns a
+ // filtered message in output.
+ FilteredMessage FilterMessageFragments(const InputSlice*, size_t num_slices);
+
+ // Helper for tests, where the input is a contiguous buffer.
+ FilteredMessage FilterMessage(const void* data, size_t len) {
+ InputSlice slice{data, len};
+ return FilterMessageFragments(&slice, 1);
+ }
+
+ // When enabled returns a map of "field path" to "usage counter".
+ // The key (std::string) is a binary buffer (i.e. NOT an ASCII/UTF-8 string)
+ // which contains a varint for each field. Consider the following:
+ // message Root { Sub1 f1 = 1; };
+ // message Sub1 { Sub2 f2 = 7;}
+ // message Sub2 { string f3 = 5; }
+ // The field .f1.f2.f3 will be encoded as \x01\0x07\x05.
+ // The value is the number of times that field has been encountered. If the
+ // field is not allow-listed in the bytecode (the field is stripped in output)
+ // the count will be negative.
+ void enable_field_usage_tracking(bool x) { track_field_usage_ = x; }
+ const std::unordered_map<std::string, int32_t>& field_usage() const {
+ return field_usage_;
+ }
+
+ // Exposed only for DCHECKS in TracingServiceImpl.
+ uint32_t root_msg_index() { return root_msg_index_; }
+
+ private:
+ // This is called by FilterMessageFragments().
+ // Inlining allows the compiler turn the per-byte call/return into a for loop,
+ // while, at the same time, keeping the code easy to read and reason about.
+ // It gives a 20-25% speedup (265ms vs 215ms for a 25MB trace).
+ void FilterOneByte(uint8_t octet) PERFETTO_ALWAYS_INLINE;
+
+ // No-inline because this is a slowpath (only when usage tracking is enabled).
+ void IncrementCurrentFieldUsage(uint32_t field_id,
+ bool allowed) PERFETTO_NO_INLINE;
+
+ // Gets into an error state which swallows all the input and emits no output.
+ void SetUnrecoverableErrorState();
+
+ // We keep track of the the nest of messages in a stack. Each StackState
+ // object corresponds to a level of nesting in the proto message structure.
+ // Every time a new field of type len-delimited that has a corresponding
+ // sub-message in the bytecode is encountered, a new StackState is pushed in
+ // |stack_|. stack_[0] is a sentinel to prevent over-popping without adding
+ // extra branches in the fastpath.
+ // |stack_|. stack_[1] is the state of the root message.
+ struct StackState {
+ uint32_t in_bytes = 0; // Number of input bytes processed.
+
+ // When |in_bytes| reaches this value, the current state should be popped.
+ // This is set when recursing into nested submessages. This is 0 only for
+ // stack_[0] (we don't know the size of the root message upfront).
+ uint32_t in_bytes_limit = 0;
+
+ // This is set when a len-delimited message is encountered, either a string
+ // or a nested submessage that is NOT allow-listed in the bytecode.
+ // This causes input bytes to be consumed without being parsed from the
+ // input stream. If |passthrough_eaten_bytes| == true, they will be copied
+ // as-is in output (e.g. in the case of an allowed string/bytes field).
+ uint32_t eat_next_bytes = 0;
+
+ // Keeps tracks of the stream_writer output counter (out_.written()) then
+ // the StackState is pushed. This is used to work out, when popping, how
+ // many bytes have been written for the current submessage.
+ uint32_t out_bytes_written_at_start = 0;
+
+ uint32_t field_id = 0; // The proto field id for the current message.
+ uint32_t msg_index = 0; // The index of the message filter in the bytecode.
+
+ // This is a pointer to the proto preamble for the current submessage
+ // (it's nullptr for stack_[0] and non-null elsewhere). This will be filled
+ // with the actual size of the message (out_.written() -
+ // |out_bytes_written_at_start|) when finishing (popping) the message.
+ // This must be filled using WriteRedundantVarint(). Note that the
+ // |size_field_len| is variable and depends on the actual length of the
+ // input message. If the output message has roughly the same size of the
+ // input message, the length will not be redundant.
+ // In other words: the length of the field is reserved when the submessage
+ // starts. At that point we know the upper-bound for the output message
+ // (a filtered submessage can be <= the original one, but not >). So we
+ // reserve as many bytes it takes to write the input length in varint.
+ // Then, when the message is finalized and we know the actual output size
+ // we backfill the field.
+ // Consider the example of a submessage where the input size = 130 (>127,
+ // 2 varint bytes) and the output is 120 bytes. The length will be 2 bytes
+ // wide even though could have been encoded with just one byte.
+ uint8_t* size_field = nullptr;
+ uint32_t size_field_len = 0;
+
+ // When true the next |eat_next_bytes| are copied as-is in output.
+ // It seems that keeping this field at the end rather than next to
+ // |eat_next_bytes| makes the filter a little (but measurably) faster.
+ // (likely something related with struct layout vs cache sizes).
+ bool passthrough_eaten_bytes = false;
+ };
+
+ uint32_t out_written() { return static_cast<uint32_t>(out_ - &out_buf_[0]); }
+
+ std::unique_ptr<uint8_t[]> out_buf_;
+ uint8_t* out_ = nullptr;
+ uint8_t* out_end_ = nullptr;
+ uint32_t root_msg_index_ = 0;
+
+ FilterBytecodeParser filter_;
+ MessageTokenizer tokenizer_;
+ std::vector<StackState> stack_;
+
+ bool error_ = false;
+ bool track_field_usage_ = false;
+ std::unordered_map<std::string, int32_t> field_usage_;
+};
+
+} // namespace protozero
+
+#endif // SRC_PROTOZERO_FILTERING_MESSAGE_FILTER_H_
diff --git a/src/protozero/filtering/message_filter_benchmark.cc b/src/protozero/filtering/message_filter_benchmark.cc
new file mode 100644
index 000000000..c800c8234
--- /dev/null
+++ b/src/protozero/filtering/message_filter_benchmark.cc
@@ -0,0 +1,48 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <benchmark/benchmark.h>
+
+#include <algorithm>
+#include <string>
+
+#include "perfetto/ext/base/file_utils.h"
+#include "src/base/test/utils.h"
+#include "src/protozero/filtering/message_filter.h"
+
+static void BM_ProtozeroMessageFilter(benchmark::State& state) {
+ std::string trace_data;
+ static const char kTestTrace[] = "test/data/example_android_trace_30s.pb";
+ perfetto::base::ReadFile(perfetto::base::GetTestDataPath(kTestTrace),
+ &trace_data);
+ PERFETTO_CHECK(!trace_data.empty());
+
+ std::string filter;
+ static const char kFullTraceFilter[] = "test/data/full_trace_filter.bytecode";
+ perfetto::base::ReadFile(kFullTraceFilter, &filter);
+ PERFETTO_CHECK(!filter.empty());
+
+ protozero::MessageFilter filt;
+ filt.LoadFilterBytecode(filter.data(), filter.size());
+
+ for (auto _ : state) {
+ auto res = filt.FilterMessage(trace_data.data(), trace_data.size());
+ benchmark::DoNotOptimize(res);
+ benchmark::ClobberMemory();
+ }
+ state.SetBytesProcessed(
+ static_cast<int64_t>(state.iterations() * trace_data.size()));
+}
+
+BENCHMARK(BM_ProtozeroMessageFilter);
diff --git a/src/protozero/filtering/message_filter_fuzzer.cc b/src/protozero/filtering/message_filter_fuzzer.cc
new file mode 100644
index 000000000..60cfe5c84
--- /dev/null
+++ b/src/protozero/filtering/message_filter_fuzzer.cc
@@ -0,0 +1,104 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "perfetto/base/logging.h"
+#include "src/protozero/filtering/message_filter.h"
+
+namespace protozero {
+namespace {
+
+// A valid filter bytecode obtained from a perfetto.protos.Trace message.
+uint8_t kValidFilter[] = {
+ 0x0b, 0x01, 0x12, 0x04, 0x00, 0x0b, 0x02, 0x13, 0x0f, 0x19, 0x23, 0x13,
+ 0x33, 0x14, 0x3b, 0x15, 0x41, 0x4b, 0x11, 0x51, 0x5b, 0x16, 0x63, 0x3b,
+ 0x69, 0x8b, 0x02, 0x21, 0x93, 0x02, 0x2a, 0x9b, 0x02, 0x2c, 0xab, 0x02,
+ 0x2e, 0xb3, 0x02, 0x09, 0xc3, 0x02, 0x30, 0xca, 0x02, 0x02, 0xdb, 0x02,
+ 0x31, 0xe3, 0x02, 0x26, 0xeb, 0x02, 0x32, 0xf3, 0x02, 0x07, 0xfb, 0x02,
+ 0x33, 0x8b, 0x03, 0x0a, 0x9b, 0x03, 0x34, 0xb3, 0x03, 0x06, 0xc3, 0x03,
+ 0x37, 0xd1, 0x03, 0xdb, 0x03, 0x3c, 0xe3, 0x03, 0x39, 0x93, 0x04, 0x38,
+ 0x9b, 0x04, 0x3a, 0x00, 0x09, 0x13, 0x03, 0x19, 0x00, 0x0a, 0x02, 0x1b,
+ 0x04, 0x23, 0x05, 0x5b, 0x06, 0x6b, 0x06, 0x83, 0x01, 0x07, 0x8b, 0x01,
+ 0x08, 0x93, 0x01, 0x07, 0x9b, 0x01, 0x07, 0xa3, 0x01, 0x08, 0x9b, 0x02,
+ 0x08, 0xcb, 0x02, 0x08, 0xd3, 0x02, 0x08, 0xdb, 0x02, 0x09, 0xe3, 0x02,
+ 0x07, 0xeb, 0x02, 0x0a, 0xf3, 0x02, 0x07, 0xfb, 0x02, 0x0b, 0x83, 0x03,
+ 0x06, 0x8b, 0x03, 0x0b, 0x93, 0x03, 0x05, 0x9b, 0x03, 0x0b, 0xab, 0x03,
+ 0x0c, 0xb3, 0x03, 0x0c, 0xbb, 0x03, 0x0c, 0x9b, 0x04, 0x0d, 0xa3, 0x04,
+ 0x07, 0xab, 0x04, 0x06, 0xb3, 0x04, 0x07, 0xc3, 0x04, 0x06, 0xcb, 0x04,
+ 0x07, 0xd3, 0x04, 0x07, 0xdb, 0x04, 0x06, 0x93, 0x07, 0x08, 0xeb, 0x07,
+ 0x08, 0xcb, 0x09, 0x07, 0xd3, 0x09, 0x05, 0xf3, 0x0b, 0x06, 0xdb, 0x0e,
+ 0x09, 0xe3, 0x0e, 0x09, 0xeb, 0x0e, 0x07, 0xf3, 0x0e, 0x09, 0xfb, 0x0e,
+ 0x09, 0x83, 0x0f, 0x07, 0x8b, 0x0f, 0x06, 0x93, 0x0f, 0x07, 0xb3, 0x0f,
+ 0x0a, 0xc3, 0x0f, 0x0e, 0xfb, 0x0f, 0x0e, 0x83, 0x10, 0x08, 0xfb, 0x10,
+ 0x08, 0x8b, 0x11, 0x08, 0xcb, 0x13, 0x06, 0xd3, 0x13, 0x07, 0xdb, 0x13,
+ 0x07, 0xb3, 0x14, 0x07, 0x00, 0x11, 0x00, 0x0a, 0x07, 0x00, 0x0a, 0x02,
+ 0x00, 0x0a, 0x03, 0x00, 0x0a, 0x05, 0x00, 0x0a, 0x04, 0x00, 0x0a, 0x06,
+ 0x00, 0x09, 0x00, 0x00, 0x0a, 0x03, 0x29, 0x00, 0x0a, 0x08, 0x00, 0x0b,
+ 0x10, 0x13, 0x07, 0x19, 0x00, 0x0a, 0x03, 0x23, 0x07, 0x29, 0x00, 0x0b,
+ 0x12, 0x11, 0x00, 0x0a, 0x0a, 0x5b, 0x07, 0x00, 0x0a, 0x02, 0x1b, 0x07,
+ 0x00, 0x0b, 0x09, 0x11, 0x00, 0x0b, 0x06, 0x13, 0x06, 0x1b, 0x0e, 0x22,
+ 0x02, 0x33, 0x06, 0x39, 0x43, 0x06, 0x49, 0x00, 0x0a, 0x03, 0x2b, 0x0b,
+ 0x33, 0x20, 0x42, 0x05, 0x82, 0x01, 0x02, 0xa1, 0x01, 0xc3, 0x01, 0x17,
+ 0xcb, 0x01, 0x04, 0xd3, 0x01, 0x0b, 0xdb, 0x01, 0x06, 0xe3, 0x01, 0x1e,
+ 0xeb, 0x01, 0x1f, 0xf2, 0x01, 0x02, 0x00, 0x0b, 0x18, 0x12, 0x0c, 0x73,
+ 0x1a, 0x7b, 0x1c, 0x83, 0x01, 0x1d, 0x8b, 0x01, 0x05, 0x00, 0x0b, 0x08,
+ 0x13, 0x19, 0x00, 0x0a, 0x2e, 0x00, 0x0a, 0x03, 0x23, 0x1b, 0x2b, 0x1b,
+ 0x33, 0x05, 0x00, 0x0a, 0x09, 0x53, 0x09, 0x00, 0x09, 0x13, 0x1b, 0x00,
+ 0x0a, 0x03, 0x23, 0x1b, 0x00, 0x09, 0x19, 0x00, 0x0a, 0x03, 0x23, 0x06,
+ 0x2a, 0x02, 0x00, 0x0a, 0x04, 0x31, 0x42, 0x08, 0x92, 0x01, 0x02, 0x00,
+ 0x0b, 0x22, 0x13, 0x23, 0x1a, 0x03, 0x33, 0x07, 0x3b, 0x09, 0x42, 0x03,
+ 0x5b, 0x0b, 0x62, 0x03, 0x81, 0x01, 0x8b, 0x01, 0x29, 0x92, 0x01, 0x02,
+ 0xa3, 0x01, 0x07, 0xab, 0x01, 0x0b, 0xb2, 0x01, 0x03, 0xcb, 0x01, 0x09,
+ 0xda, 0x01, 0x02, 0x00, 0x09, 0x21, 0x00, 0x0b, 0x24, 0x11, 0x00, 0x0a,
+ 0x04, 0x31, 0xa3, 0x06, 0x25, 0xbb, 0x06, 0x26, 0xc3, 0x06, 0x0a, 0xcb,
+ 0x06, 0x27, 0xd3, 0x06, 0x06, 0xeb, 0x06, 0x0b, 0x00, 0x0a, 0x03, 0x52,
+ 0x02, 0x00, 0x0a, 0x04, 0x32, 0x03, 0x00, 0x0a, 0x02, 0x22, 0x02, 0x33,
+ 0x28, 0x3a, 0x02, 0x00, 0x2a, 0x02, 0x00, 0x09, 0x13, 0x07, 0x19, 0x00,
+ 0x09, 0x13, 0x2b, 0x00, 0x0a, 0x09, 0x00, 0x0b, 0x2d, 0x12, 0x08, 0x00,
+ 0x0a, 0x12, 0x00, 0x0b, 0x06, 0x13, 0x09, 0x1b, 0x06, 0x23, 0x05, 0x2b,
+ 0x2f, 0x32, 0x02, 0x00, 0x09, 0x13, 0x0a, 0x00, 0x0b, 0x09, 0x13, 0x07,
+ 0x00, 0x09, 0x1a, 0x03, 0x00, 0x0b, 0x09, 0x12, 0x02, 0x00, 0x0b, 0x08,
+ 0x12, 0x02, 0x00, 0x0b, 0x35, 0x11, 0x00, 0x0b, 0x36, 0x13, 0x36, 0x00,
+ 0x09, 0x13, 0x07, 0x1b, 0x0b, 0x00, 0x09, 0x13, 0x08, 0x1b, 0x06, 0x23,
+ 0x06, 0x2a, 0x02, 0x3b, 0x06, 0x00, 0x0a, 0x05, 0x82, 0x01, 0x03, 0x00,
+ 0x09, 0x1b, 0x31, 0x23, 0x26, 0x29, 0x33, 0x09, 0x3b, 0x06, 0x43, 0x31,
+ 0x00, 0x0b, 0x06, 0x00, 0x0b, 0x06, 0x13, 0x06, 0x23, 0x09, 0x2b, 0x06,
+ 0x33, 0x09, 0x3b, 0x06, 0x83, 0x01, 0x06, 0x8b, 0x01, 0x06, 0x93, 0x01,
+ 0x06, 0x9b, 0x01, 0x05, 0x00, 0x5b, 0x3d, 0xd1, 0x03, 0x00, 0x59, 0xf9,
+ 0x01, 0x00, 0x8f, 0xf8, 0xf5, 0xcb, 0x06};
+
+int FuzzMessageFilter(const uint8_t* data, size_t size) {
+ MessageFilter filter;
+ PERFETTO_CHECK(filter.LoadFilterBytecode(kValidFilter, sizeof(kValidFilter)));
+
+ auto res = filter.FilterMessage(data, size);
+
+ // Either parsing fails or if it succeeds, the output data must be <= input.
+ PERFETTO_CHECK(res.error || res.size <= size);
+ return 0;
+}
+
+} // namespace
+} // namespace protozero
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size);
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+ return protozero::FuzzMessageFilter(data, size);
+}
diff --git a/src/protozero/filtering/message_filter_unittest.cc b/src/protozero/filtering/message_filter_unittest.cc
new file mode 100644
index 000000000..83cb911f2
--- /dev/null
+++ b/src/protozero/filtering/message_filter_unittest.cc
@@ -0,0 +1,810 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "test/gtest_and_gmock.h"
+
+#include <random>
+
+#include "perfetto/ext/base/file_utils.h"
+#include "perfetto/ext/base/string_utils.h"
+#include "perfetto/ext/base/temp_file.h"
+#include "perfetto/protozero/proto_decoder.h"
+#include "perfetto/protozero/proto_utils.h"
+#include "perfetto/protozero/scattered_heap_buffer.h"
+#include "protos/perfetto/trace/trace.pb.h"
+#include "src/protozero/filtering/filter_util.h"
+#include "src/protozero/filtering/message_filter.h"
+
+namespace protozero {
+
+namespace {
+
+TEST(MessageFilterTest, EndToEnd) {
+ auto schema = perfetto::base::TempFile::Create();
+ static const char kSchema[] = R"(
+ syntax = "proto2";
+ message FilterSchema {
+ message Nested {
+ optional fixed32 f32 = 2;
+ repeated string ss = 5;
+ }
+ optional int32 i32 = 1;
+ optional string str = 3;
+ repeated Nested nest = 6;
+ repeated int32 f11 = 11;
+ repeated int64 f12 = 12;
+ repeated sint32 f13 = 13;
+ repeated sint64 f14 = 14;
+ repeated fixed32 f15 = 15;
+ repeated fixed32 f16 = 16;
+ repeated fixed64 f17 = 17;
+ repeated fixed64 f18 = 18;
+ repeated float f19 = 19;
+ repeated double f20 = 20;
+ };
+ )";
+
+ perfetto::base::WriteAll(*schema, kSchema, strlen(kSchema));
+ perfetto::base::FlushFile(*schema);
+
+ FilterUtil filter;
+ ASSERT_TRUE(filter.LoadMessageDefinition(schema.path(), "", ""));
+ std::string bytecode = filter.GenerateFilterBytecode();
+ ASSERT_GT(bytecode.size(), 0u);
+
+ HeapBuffered<Message> msg;
+ msg->AppendVarInt(/*field_id=*/1, -1000000000ll);
+ msg->AppendVarInt(/*field_id=*/2, 42);
+ msg->AppendString(/*field_id=*/3, "foobar");
+ msg->AppendFixed(/*field_id=*/4, 10);
+ msg->AppendVarInt(/*field_id=*/11, INT32_MIN);
+ msg->AppendVarInt(/*field_id=*/12, INT64_MIN);
+ msg->AppendSignedVarInt(/*field_id=*/13, INT32_MIN);
+ msg->AppendSignedVarInt(/*field_id=*/14, INT64_MIN);
+ msg->AppendFixed(/*field_id=*/15, static_cast<int32_t>(INT32_MIN));
+ msg->AppendFixed(/*field_id=*/16, static_cast<int32_t>(INT32_MAX));
+ msg->AppendFixed(/*field_id=*/17, static_cast<int64_t>(INT64_MIN));
+ msg->AppendFixed(/*field_id=*/18, static_cast<int64_t>(INT64_MAX));
+ msg->AppendFixed(/*field_id=*/19, FLT_EPSILON);
+ msg->AppendFixed(/*field_id=*/20, DBL_EPSILON);
+
+ auto* nest = msg->BeginNestedMessage<Message>(/*field_id=*/6);
+ nest->AppendFixed(/*field_id=*/1, 10);
+ nest->AppendFixed(/*field_id=*/2, static_cast<int32_t>(-2000000000ll));
+ nest->AppendString(/*field_id=*/4, "stripped");
+ nest->AppendString(/*field_id=*/5, "");
+ nest->Finalize();
+
+ MessageFilter flt;
+ ASSERT_TRUE(flt.LoadFilterBytecode(bytecode.data(), bytecode.size()));
+
+ std::vector<uint8_t> encoded = msg.SerializeAsArray();
+
+ for (int repetitions = 0; repetitions < 3; ++repetitions) {
+ auto filtered = flt.FilterMessage(encoded.data(), encoded.size());
+ ASSERT_LT(filtered.size, encoded.size());
+
+ ProtoDecoder dec(filtered.data.get(), filtered.size);
+ EXPECT_TRUE(dec.FindField(1).valid());
+ EXPECT_EQ(dec.FindField(1).as_int64(), -1000000000ll);
+ EXPECT_FALSE(dec.FindField(2).valid());
+ EXPECT_TRUE(dec.FindField(3).valid());
+ EXPECT_EQ(dec.FindField(3).as_std_string(), "foobar");
+ EXPECT_FALSE(dec.FindField(4).valid());
+ EXPECT_TRUE(dec.FindField(6).valid());
+ for (uint32_t i = 11; i <= 20; ++i)
+ EXPECT_TRUE(dec.FindField(i).valid());
+
+ EXPECT_EQ(dec.FindField(11).as_int32(), INT32_MIN);
+ EXPECT_EQ(dec.FindField(12).as_int64(), INT64_MIN);
+ EXPECT_EQ(dec.FindField(13).as_sint32(), INT32_MIN);
+ EXPECT_EQ(dec.FindField(14).as_sint64(), INT64_MIN);
+ EXPECT_EQ(dec.FindField(15).as_int32(), INT32_MIN);
+ EXPECT_EQ(dec.FindField(16).as_int32(), INT32_MAX);
+ EXPECT_EQ(dec.FindField(17).as_int64(), INT64_MIN);
+ EXPECT_EQ(dec.FindField(18).as_int64(), INT64_MAX);
+ EXPECT_EQ(dec.FindField(19).as_float(), FLT_EPSILON);
+ EXPECT_EQ(dec.FindField(20).as_double(), DBL_EPSILON);
+
+ ProtoDecoder nest_dec(dec.FindField(6).as_bytes());
+ EXPECT_FALSE(nest_dec.FindField(1).valid());
+ EXPECT_TRUE(nest_dec.FindField(2).valid());
+ EXPECT_EQ(nest_dec.FindField(2).as_int32(), -2000000000ll);
+ EXPECT_TRUE(nest_dec.FindField(5).valid());
+ EXPECT_EQ(nest_dec.FindField(5).as_bytes().size, 0u);
+ }
+}
+
+TEST(MessageFilterTest, ChangeRoot) {
+ auto schema = perfetto::base::TempFile::Create();
+ static const char kSchema[] = R"(
+ syntax = "proto2";
+ message FilterSchema {
+ message Nested {
+ message Nested2 {
+ optional int32 e = 5;
+ }
+ optional int32 c = 3;
+ repeated Nested2 d = 4;
+ }
+ optional int32 a = 1;
+ optional Nested b = 2;
+ };
+ )";
+
+ perfetto::base::WriteAll(*schema, kSchema, strlen(kSchema));
+ perfetto::base::FlushFile(*schema);
+
+ FilterUtil filter;
+ ASSERT_TRUE(filter.LoadMessageDefinition(schema.path(), "", ""));
+ std::string bytecode = filter.GenerateFilterBytecode();
+ ASSERT_GT(bytecode.size(), 0u);
+
+ HeapBuffered<Message> msg;
+ msg->AppendVarInt(/*field_id=*/1, 101);
+ msg->AppendVarInt(/*field_id=*/3, 103);
+ msg->AppendVarInt(/*field_id=*/5, 105);
+ auto* nest = msg->BeginNestedMessage<Message>(/*field_id=*/4); // Nested b.
+ nest->AppendVarInt(/*field_id=*/5, 205);
+ nest->Finalize();
+ std::vector<uint8_t> encoded = msg.SerializeAsArray();
+
+ MessageFilter flt;
+ ASSERT_TRUE(flt.LoadFilterBytecode(bytecode.data(), bytecode.size()));
+ uint32_t roots[2]{2, 4};
+
+ // First set the root to field id ".2" (.b). The fliter should happen treating
+ // |Nested| as rot, so allowing only field 3 and 4 (Nested2) through.
+ {
+ flt.SetFilterRoot(roots, 1);
+ auto filtered = flt.FilterMessage(encoded.data(), encoded.size());
+ ASSERT_LT(filtered.size, encoded.size());
+ ProtoDecoder dec(filtered.data.get(), filtered.size);
+ EXPECT_FALSE(dec.FindField(1).valid());
+ EXPECT_TRUE(dec.FindField(3).valid());
+ EXPECT_EQ(dec.FindField(3).as_int32(), 103);
+ EXPECT_FALSE(dec.FindField(5).valid());
+ EXPECT_TRUE(dec.FindField(4).valid());
+ EXPECT_EQ(dec.FindField(4).as_std_string(), "(\xCD\x01");
+ }
+
+ // Now set the root to ".2.4" (.b.d). This should allow only the field "e"
+ // to pass through.
+ {
+ flt.SetFilterRoot(roots, 2);
+ auto filtered = flt.FilterMessage(encoded.data(), encoded.size());
+ ASSERT_LT(filtered.size, encoded.size());
+ ProtoDecoder dec(filtered.data.get(), filtered.size);
+ EXPECT_FALSE(dec.FindField(1).valid());
+ EXPECT_FALSE(dec.FindField(3).valid());
+ EXPECT_FALSE(dec.FindField(4).valid());
+ EXPECT_TRUE(dec.FindField(5).valid());
+ EXPECT_EQ(dec.FindField(5).as_int32(), 105);
+ }
+}
+
+TEST(MessageFilterTest, MalformedInput) {
+ // Create and load a simple filter.
+ auto schema = perfetto::base::TempFile::Create();
+ static const char kSchema[] = R"(
+ syntax = "proto2";
+ message FilterSchema {
+ message Nested {
+ optional fixed32 f32 = 4;
+ repeated string ss = 5;
+ }
+ optional int32 i32 = 1;
+ optional string str = 2;
+ repeated Nested nest = 3;
+ };
+ )";
+ perfetto::base::WriteAll(*schema, kSchema, strlen(kSchema));
+ perfetto::base::FlushFile(*schema);
+ FilterUtil filter;
+ ASSERT_TRUE(filter.LoadMessageDefinition(schema.path(), "", ""));
+ std::string bytecode = filter.GenerateFilterBytecode();
+ ASSERT_GT(bytecode.size(), 0u);
+ MessageFilter flt;
+ ASSERT_TRUE(flt.LoadFilterBytecode(bytecode.data(), bytecode.size()));
+
+ {
+ // A malformed message found by the fuzzer.
+ static const uint8_t kData[]{
+ 0x52, 0x21, // ID=10, type=len-delimited, len=33.
+ 0xa0, 0xa4, // Early terminating payload.
+ };
+ auto res = flt.FilterMessage(kData, sizeof(kData));
+ EXPECT_TRUE(res.error);
+ }
+
+ {
+ // A malformed message which contains a non-terminated varint.
+ static const uint8_t kData[]{
+ 0x08, 0x2A, // A valid varint field id=1 value=42 (0x2A).
+ 0x08, 0xFF, // An unterminated varint.
+ };
+ auto res = flt.FilterMessage(kData, sizeof(kData));
+ EXPECT_TRUE(res.error);
+ }
+
+ {
+ // A malformed message which contains a sub-message with a field that brings
+ // it out of the outer size.
+ static const uint8_t kData[]{
+ 0x08, 0x2A, // A valid varint field id=1 value=42 (0x2A).
+ 0x1A, 0x04, // A len-delim field, id=3, length=4.
+ // The nested message |nest| starts here.
+ 0x25, 0x0, 0x0, 0x0, 0x01, // A fixed32 field, id=4.
+ // Note that the fixed32 field has an expected length of 4 but that
+ // overflows the size of the |nest| method, because with its 0x25
+ // preamble it becomes 5 bytes. At this point this should cause a
+ // persistent failure.
+ };
+ auto res = flt.FilterMessage(kData, sizeof(kData));
+ EXPECT_TRUE(res.error);
+ }
+
+ // A parsing failure shoulnd't affect the ability to filter the following
+ // message. Try again but this time with a valid message.
+ {
+ static const uint8_t kData[]{
+ 0x08, 0x2A, // A valid varint field id=1 value=42 (0x2A).
+ 0x1A, 0x05, // A len-delim field, id=3, length=5.
+ 0x25, 0x0, 0x0, 0x0, 0x01, // A fixed32 field, id=4.
+ 0x38, 0x42, // A valid but not allowed varint field id=7.
+ };
+ auto res = flt.FilterMessage(kData, sizeof(kData));
+ EXPECT_FALSE(res.error);
+ EXPECT_EQ(res.size, sizeof(kData) - 2); // last 2 bytes should be skipped.
+ EXPECT_EQ(memcmp(kData, res.data.get(), res.size), 0);
+ }
+}
+
+// It processes a real test trace with a real filter. The filter has been
+// obtained from the full upstream perfetto proto (+ re-adding the for_testing
+// field which got removed after adding most test traces). This covers the most
+// complex case of filtering a real trace with a filter that allows all possible
+// fields, hence re-entering deeply in most nested fields.
+TEST(MessageFilterTest, RealTracePassthrough) {
+ // This is test/data/android_log_ring_buffer_mode.pb. It's re-encoded as a
+ // constant because unittests cannot depend on test/data/, only integration
+ // tests can.
+ static const uint8_t kTraceData[]{
+ 0x0a, 0x16, 0x18, 0x8f, 0x4e, 0xa2, 0x02, 0x10, 0x82, 0x47, 0x7a, 0x76,
+ 0xb2, 0x8d, 0x42, 0xba, 0x81, 0xdc, 0x33, 0x32, 0x6d, 0x57, 0xa0, 0x79,
+ 0x0a, 0x5f, 0x18, 0x8f, 0x4e, 0x32, 0x5a, 0x0a, 0x09, 0x08, 0x06, 0x10,
+ 0xf4, 0xd3, 0xea, 0xbb, 0xba, 0x55, 0x0a, 0x0c, 0x08, 0x02, 0x10, 0xf9,
+ 0xcc, 0xb4, 0xd1, 0xe8, 0xdc, 0xa5, 0xbc, 0x15, 0x0a, 0x09, 0x08, 0x04,
+ 0x10, 0x86, 0xb9, 0x9c, 0xba, 0xba, 0x55, 0x0a, 0x0c, 0x08, 0x01, 0x10,
+ 0xeb, 0xe9, 0x82, 0xd3, 0xe8, 0xdc, 0xa5, 0xbc, 0x15, 0x0a, 0x09, 0x08,
+ 0x03, 0x10, 0xac, 0xd6, 0xea, 0xbb, 0xba, 0x55, 0x0a, 0x09, 0x08, 0x05,
+ 0x10, 0x9b, 0xe1, 0xd8, 0xbb, 0xba, 0x55, 0x0a, 0x07, 0x08, 0x07, 0x10,
+ 0xf5, 0xe6, 0xd9, 0x55, 0x0a, 0x07, 0x08, 0x08, 0x10, 0xc1, 0xcc, 0xa7,
+ 0x41, 0x0a, 0x27, 0x18, 0x8f, 0x4e, 0x9a, 0x02, 0x21, 0x0a, 0x13, 0x08,
+ 0xf0, 0x1f, 0x10, 0x01, 0x18, 0x00, 0x20, 0x00, 0x28, 0x00, 0x30, 0x00,
+ 0x38, 0x00, 0x40, 0x00, 0x48, 0x00, 0x10, 0x01, 0x18, 0x07, 0x20, 0x06,
+ 0x28, 0x0b, 0x30, 0x01, 0x38, 0x01, 0x0a, 0xd5, 0x01, 0x18, 0x8f, 0x4e,
+ 0x8a, 0x02, 0xce, 0x01, 0x0a, 0x06, 0x08, 0x80, 0x80, 0x02, 0x20, 0x00,
+ 0x12, 0xa5, 0x01, 0x0a, 0xa2, 0x01, 0x0a, 0x0b, 0x61, 0x6e, 0x64, 0x72,
+ 0x6f, 0x69, 0x64, 0x2e, 0x6c, 0x6f, 0x67, 0x10, 0x00, 0x18, 0x00, 0x20,
+ 0x00, 0xa2, 0x06, 0x04, 0x50, 0x00, 0x58, 0x00, 0xaa, 0x06, 0x02, 0x0a,
+ 0x00, 0xb2, 0x06, 0x08, 0x08, 0x00, 0x10, 0x00, 0x18, 0x00, 0x20, 0x00,
+ 0xba, 0x06, 0x06, 0x10, 0x00, 0x18, 0x00, 0x20, 0x00, 0xc2, 0x06, 0x06,
+ 0x08, 0x00, 0x18, 0x00, 0x28, 0x00, 0xca, 0x06, 0x0a, 0x08, 0x00, 0x28,
+ 0x00, 0x32, 0x04, 0x28, 0x00, 0x30, 0x00, 0xd2, 0x06, 0x02, 0x08, 0x00,
+ 0xda, 0x06, 0x02, 0x18, 0x00, 0xc2, 0x3e, 0x00, 0xfa, 0xff, 0xff, 0xff,
+ 0x07, 0x46, 0x08, 0x00, 0x10, 0x00, 0x18, 0x00, 0x20, 0x00, 0x28, 0x00,
+ 0x32, 0x3a, 0x08, 0x00, 0x10, 0x00, 0x18, 0x00, 0x20, 0x00, 0x29, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x00, 0x45, 0x00, 0x00,
+ 0x00, 0x00, 0x49, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55,
+ 0x00, 0x00, 0x00, 0x00, 0x58, 0x00, 0x60, 0x00, 0x6a, 0x00, 0x72, 0x00,
+ 0x18, 0xe0, 0xd4, 0x03, 0x20, 0x00, 0x28, 0x00, 0x3a, 0x06, 0x08, 0x00,
+ 0x10, 0x00, 0x18, 0x00, 0x40, 0x00, 0x48, 0x00, 0x50, 0x00, 0x5a, 0x02,
+ 0x08, 0x00, 0x60, 0x00, 0x68, 0x00, 0x0a, 0x94, 0x01, 0x40, 0xd9, 0xf4,
+ 0x98, 0x96, 0xbe, 0x54, 0xba, 0x02, 0x84, 0x81, 0x80, 0x00, 0x0a, 0xff,
+ 0x80, 0x80, 0x00, 0x38, 0x04, 0x32, 0x08, 0x70, 0x65, 0x72, 0x66, 0x65,
+ 0x74, 0x74, 0x6f, 0x42, 0x5d, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64,
+ 0x5f, 0x6c, 0x6f, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x6f,
+ 0x75, 0x72, 0x63, 0x65, 0x2e, 0x63, 0x63, 0x3a, 0x31, 0x35, 0x35, 0x20,
+ 0x53, 0x74, 0x61, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x41, 0x6e, 0x64,
+ 0x72, 0x6f, 0x69, 0x64, 0x20, 0x6c, 0x6f, 0x67, 0x20, 0x64, 0x61, 0x74,
+ 0x61, 0x20, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x3a, 0x20, 0x73, 0x74,
+ 0x72, 0x65, 0x61, 0x6d, 0x20, 0x74, 0x61, 0x69, 0x6c, 0x3d, 0x31, 0x20,
+ 0x6c, 0x69, 0x64, 0x73, 0x3d, 0x30, 0x2c, 0x32, 0x2c, 0x33, 0x2c, 0x34,
+ 0x2c, 0x37, 0x28, 0xdb, 0xe6, 0x9b, 0xfb, 0xeb, 0xdb, 0xa5, 0xbc, 0x15,
+ 0x08, 0x00, 0x10, 0xb5, 0x58, 0x18, 0xb5, 0x58, 0x20, 0x00, 0x18, 0x8f,
+ 0x4e, 0x0a, 0xf4, 0x0a, 0x40, 0xb2, 0x84, 0xde, 0xdd, 0x8f, 0x55, 0xba,
+ 0x02, 0xe4, 0x8a, 0x80, 0x00, 0x0a, 0xe4, 0x80, 0x80, 0x00, 0x38, 0x04,
+ 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74,
+ 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x38, 0x72, 0x6d,
+ 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f,
+ 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x63, 0x62, 0x3a, 0x20, 0x63, 0x6c,
+ 0x6e, 0x74, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x31, 0x34, 0x20, 0x63, 0x6f,
+ 0x6e, 0x6e, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x37, 0x32, 0x31, 0x64, 0x61,
+ 0x30, 0x37, 0x31, 0x30, 0x30, 0x0a, 0x28, 0x92, 0xad, 0xf6, 0xf0, 0xbd,
+ 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, 0xe9, 0x06,
+ 0x20, 0x00, 0x0a, 0xf3, 0x80, 0x80, 0x00, 0x38, 0x06, 0x32, 0x0b, 0x72,
+ 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x4e,
+ 0x49, 0x4e, 0x46, 0x4f, 0x3a, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f,
+ 0x72, 0x61, 0x67, 0x65, 0x5f, 0x72, 0x77, 0x5f, 0x69, 0x6f, 0x76, 0x65,
+ 0x63, 0x5f, 0x63, 0x62, 0x3a, 0x20, 0x57, 0x72, 0x69, 0x74, 0x65, 0x20,
+ 0x69, 0x6f, 0x76, 0x65, 0x63, 0x20, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73,
+ 0x74, 0x20, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x20, 0x66,
+ 0x6f, 0x72, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64,
+ 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x63, 0x28, 0x9e, 0xa1, 0xea, 0xf0, 0xbd,
+ 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x07, 0x10, 0xe9, 0x06, 0x18, 0xe9, 0x06,
+ 0x20, 0x00, 0x0a, 0xfd, 0x80, 0x80, 0x00, 0x38, 0x06, 0x32, 0x0b, 0x72,
+ 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x58,
+ 0x49, 0x4e, 0x46, 0x4f, 0x3a, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f,
+ 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f,
+ 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, 0x20, 0x43, 0x61, 0x6c, 0x6c,
+ 0x69, 0x6e, 0x67, 0x20, 0x57, 0x72, 0x69, 0x74, 0x65, 0x20, 0x5b, 0x6f,
+ 0x66, 0x66, 0x73, 0x65, 0x74, 0x3d, 0x30, 0x2c, 0x20, 0x73, 0x69, 0x7a,
+ 0x65, 0x3d, 0x36, 0x35, 0x35, 0x33, 0x36, 0x5d, 0x66, 0x6f, 0x72, 0x20,
+ 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f,
+ 0x66, 0x73, 0x63, 0x21, 0x28, 0xe6, 0xc4, 0x80, 0xf1, 0xbd, 0xdc, 0xa5,
+ 0xbc, 0x15, 0x08, 0x07, 0x10, 0xf6, 0x0b, 0x18, 0xf6, 0x0b, 0x20, 0x00,
+ 0x0a, 0x80, 0x81, 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, 0x65, 0x6e,
+ 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72,
+ 0x61, 0x67, 0x65, 0x42, 0x54, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f,
+ 0x72, 0x61, 0x67, 0x65, 0x5f, 0x72, 0x77, 0x5f, 0x69, 0x6f, 0x76, 0x65,
+ 0x63, 0x5f, 0x63, 0x62, 0x3a, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f,
+ 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x63, 0x3a, 0x20, 0x72,
+ 0x65, 0x71, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x31, 0x34, 0x20, 0x6d, 0x73,
+ 0x67, 0x5f, 0x69, 0x64, 0x3d, 0x33, 0x3a, 0x20, 0x52, 0x2f, 0x57, 0x20,
+ 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x20, 0x72, 0x65, 0x63, 0x65,
+ 0x69, 0x76, 0x65, 0x64, 0x0a, 0x28, 0xc3, 0xe0, 0xf9, 0xf0, 0xbd, 0xdc,
+ 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, 0xe9, 0x06, 0x20,
+ 0x00, 0x0a, 0xc1, 0x80, 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, 0x65,
+ 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f,
+ 0x72, 0x61, 0x67, 0x65, 0x42, 0x15, 0x77, 0x61, 0x6b, 0x65, 0x6c, 0x6f,
+ 0x63, 0x6b, 0x20, 0x61, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x3a,
+ 0x20, 0x31, 0x0a, 0x28, 0xb4, 0xc7, 0x8a, 0xf1, 0xbd, 0xdc, 0xa5, 0xbc,
+ 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, 0xe9, 0x06, 0x20, 0x00, 0x0a,
+ 0x84, 0x81, 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64,
+ 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61,
+ 0x67, 0x65, 0x42, 0x58, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72,
+ 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74,
+ 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74,
+ 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x63, 0x3a, 0x20,
+ 0x55, 0x6e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x20, 0x77, 0x6f, 0x72, 0x6b,
+ 0x65, 0x72, 0x20, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x20, 0x28, 0x74,
+ 0x68, 0x5f, 0x69, 0x64, 0x3a, 0x20, 0x34, 0x39, 0x30, 0x31, 0x31, 0x34,
+ 0x39, 0x35, 0x34, 0x34, 0x36, 0x34, 0x29, 0x0a, 0x28, 0x8e, 0xe0, 0x8e,
+ 0xf1, 0xbd, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18,
+ 0xf6, 0x0b, 0x20, 0x00, 0x0a, 0x83, 0x81, 0x80, 0x00, 0x38, 0x04, 0x32,
+ 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f,
+ 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x57, 0x72, 0x6d, 0x74,
+ 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69,
+ 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, 0x20,
+ 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f,
+ 0x66, 0x73, 0x63, 0x3a, 0x20, 0x72, 0x65, 0x71, 0x5f, 0x68, 0x3d, 0x30,
+ 0x78, 0x31, 0x34, 0x20, 0x6d, 0x73, 0x67, 0x5f, 0x69, 0x64, 0x3d, 0x33,
+ 0x3a, 0x20, 0x42, 0x79, 0x74, 0x65, 0x73, 0x20, 0x77, 0x72, 0x69, 0x74,
+ 0x74, 0x65, 0x6e, 0x20, 0x3d, 0x20, 0x36, 0x35, 0x35, 0x33, 0x36, 0x0a,
+ 0x28, 0x8a, 0xe1, 0xa0, 0xf2, 0xbd, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00,
+ 0x10, 0xe9, 0x06, 0x18, 0xf6, 0x0b, 0x20, 0x00, 0x0a, 0x88, 0x81, 0x80,
+ 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e,
+ 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42,
+ 0x5c, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65,
+ 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65,
+ 0x61, 0x64, 0x3a, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f,
+ 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x63, 0x3a, 0x20, 0x72, 0x65, 0x71,
+ 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x31, 0x34, 0x20, 0x6d, 0x73, 0x67, 0x5f,
+ 0x69, 0x64, 0x3d, 0x33, 0x3a, 0x20, 0x53, 0x65, 0x6e, 0x64, 0x20, 0x72,
+ 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x3a, 0x20, 0x72, 0x65, 0x73,
+ 0x3d, 0x30, 0x20, 0x65, 0x72, 0x72, 0x3d, 0x30, 0x0a, 0x28, 0xf8, 0x89,
+ 0xa2, 0xf2, 0xbd, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06,
+ 0x18, 0xf6, 0x0b, 0x20, 0x00, 0x0a, 0xae, 0x81, 0x80, 0x00, 0x38, 0x04,
+ 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74,
+ 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x81, 0x01, 0x72,
+ 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63,
+ 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64,
+ 0x3a, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65,
+ 0x6d, 0x5f, 0x66, 0x73, 0x63, 0x3a, 0x20, 0x41, 0x62, 0x6f, 0x75, 0x74,
+ 0x20, 0x74, 0x6f, 0x20, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x20, 0x72, 0x6d,
+ 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x20, 0x63, 0x6c,
+ 0x69, 0x65, 0x6e, 0x74, 0x20, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x20,
+ 0x28, 0x74, 0x68, 0x5f, 0x69, 0x64, 0x3a, 0x20, 0x34, 0x39, 0x30, 0x31,
+ 0x31, 0x34, 0x39, 0x35, 0x34, 0x34, 0x36, 0x34, 0x29, 0x20, 0x77, 0x61,
+ 0x6b, 0x65, 0x6c, 0x6f, 0x63, 0x6b, 0x20, 0x72, 0x65, 0x6c, 0x65, 0x61,
+ 0x73, 0x65, 0x64, 0x3a, 0x20, 0x31, 0x0a, 0x0a, 0x28, 0xb9, 0xcb, 0xa9,
+ 0xf2, 0xbd, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18,
+ 0xf6, 0x0b, 0x20, 0x00, 0x0a, 0xf4, 0x80, 0x80, 0x00, 0x38, 0x06, 0x32,
+ 0x0b, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65,
+ 0x42, 0x4f, 0x49, 0x4e, 0x46, 0x4f, 0x3a, 0x72, 0x6d, 0x74, 0x5f, 0x73,
+ 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e,
+ 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, 0x20, 0x44, 0x6f,
+ 0x6e, 0x65, 0x20, 0x57, 0x72, 0x69, 0x74, 0x65, 0x20, 0x28, 0x62, 0x79,
+ 0x74, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x36, 0x35, 0x35, 0x33, 0x36, 0x29,
+ 0x20, 0x66, 0x6f, 0x72, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d,
+ 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x63, 0x21, 0x28, 0xd6, 0xbd,
+ 0x8f, 0xf2, 0xbd, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x07, 0x10, 0xf6, 0x0b,
+ 0x18, 0xf6, 0x0b, 0x20, 0x00, 0x0a, 0xe7, 0x80, 0x80, 0x00, 0x38, 0x04,
+ 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74,
+ 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x3b, 0x72, 0x6d,
+ 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x64, 0x69,
+ 0x73, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x63, 0x62, 0x3a,
+ 0x20, 0x63, 0x6c, 0x6e, 0x74, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x31, 0x34,
+ 0x20, 0x63, 0x6f, 0x6e, 0x6e, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x37, 0x32,
+ 0x31, 0x64, 0x61, 0x30, 0x37, 0x31, 0x30, 0x30, 0x0a, 0x28, 0x92, 0xd5,
+ 0xc7, 0xf2, 0xbd, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06,
+ 0x18, 0xe9, 0x06, 0x20, 0x00, 0x18, 0x8f, 0x4e, 0x0a, 0x70, 0x40, 0x92,
+ 0xaf, 0xd5, 0x8d, 0x90, 0x55, 0xba, 0x02, 0xe0, 0x80, 0x80, 0x00, 0x0a,
+ 0xdb, 0x80, 0x80, 0x00, 0x38, 0x04, 0x32, 0x08, 0x70, 0x65, 0x72, 0x66,
+ 0x65, 0x74, 0x74, 0x6f, 0x42, 0x39, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69,
+ 0x64, 0x5f, 0x6c, 0x6f, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73,
+ 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x63, 0x63, 0x3a, 0x32, 0x39, 0x31,
+ 0x20, 0x53, 0x65, 0x65, 0x6e, 0x20, 0x31, 0x31, 0x20, 0x41, 0x6e, 0x64,
+ 0x72, 0x6f, 0x69, 0x64, 0x20, 0x6c, 0x6f, 0x67, 0x20, 0x65, 0x76, 0x65,
+ 0x6e, 0x74, 0x73, 0x28, 0xd0, 0x8f, 0xfa, 0xf4, 0xbd, 0xdc, 0xa5, 0xbc,
+ 0x15, 0x08, 0x00, 0x10, 0xb5, 0x58, 0x18, 0xb5, 0x58, 0x20, 0x00, 0x18,
+ 0x8f, 0x4e, 0x0a, 0xfa, 0x0a, 0x40, 0x9c, 0xc7, 0xf8, 0xbc, 0x90, 0x55,
+ 0xba, 0x02, 0xea, 0x8a, 0x80, 0x00, 0x0a, 0xe4, 0x80, 0x80, 0x00, 0x38,
+ 0x04, 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d,
+ 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x38, 0x72,
+ 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63,
+ 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x63, 0x62, 0x3a, 0x20, 0x63,
+ 0x6c, 0x6e, 0x74, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x31, 0x35, 0x20, 0x63,
+ 0x6f, 0x6e, 0x6e, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x37, 0x32, 0x31, 0x64,
+ 0x61, 0x30, 0x37, 0x31, 0x30, 0x30, 0x0a, 0x28, 0xb4, 0xa2, 0x8b, 0xa5,
+ 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, 0xe9,
+ 0x06, 0x20, 0x00, 0x0a, 0x80, 0x81, 0x80, 0x00, 0x38, 0x04, 0x32, 0x12,
+ 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73,
+ 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x54, 0x72, 0x6d, 0x74, 0x5f,
+ 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x72, 0x77, 0x5f, 0x69,
+ 0x6f, 0x76, 0x65, 0x63, 0x5f, 0x63, 0x62, 0x3a, 0x20, 0x2f, 0x62, 0x6f,
+ 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x31,
+ 0x3a, 0x20, 0x72, 0x65, 0x71, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x31, 0x35,
+ 0x20, 0x6d, 0x73, 0x67, 0x5f, 0x69, 0x64, 0x3d, 0x33, 0x3a, 0x20, 0x52,
+ 0x2f, 0x57, 0x20, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x20, 0x72,
+ 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x0a, 0x28, 0x87, 0xf1, 0x8e,
+ 0xa5, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18,
+ 0xe9, 0x06, 0x20, 0x00, 0x0a, 0xc1, 0x80, 0x80, 0x00, 0x38, 0x04, 0x32,
+ 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f,
+ 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x15, 0x77, 0x61, 0x6b,
+ 0x65, 0x6c, 0x6f, 0x63, 0x6b, 0x20, 0x61, 0x63, 0x71, 0x75, 0x69, 0x72,
+ 0x65, 0x64, 0x3a, 0x20, 0x31, 0x0a, 0x28, 0x8e, 0x8e, 0x9e, 0xa5, 0xbe,
+ 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, 0xe9, 0x06,
+ 0x20, 0x00, 0x0a, 0x84, 0x81, 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, 0x76,
+ 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74,
+ 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x58, 0x72, 0x6d, 0x74, 0x5f, 0x73,
+ 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e,
+ 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, 0x20, 0x2f, 0x62,
+ 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73,
+ 0x31, 0x3a, 0x20, 0x55, 0x6e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x20, 0x77,
+ 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x20, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64,
+ 0x20, 0x28, 0x74, 0x68, 0x5f, 0x69, 0x64, 0x3a, 0x20, 0x34, 0x39, 0x30,
+ 0x31, 0x32, 0x38, 0x37, 0x30, 0x30, 0x36, 0x34, 0x30, 0x29, 0x0a, 0x28,
+ 0x9a, 0xa4, 0xa1, 0xa5, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10,
+ 0xe9, 0x06, 0x18, 0xf1, 0x0b, 0x20, 0x00, 0x0a, 0xf3, 0x80, 0x80, 0x00,
+ 0x38, 0x06, 0x32, 0x0b, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72,
+ 0x61, 0x67, 0x65, 0x42, 0x4e, 0x49, 0x4e, 0x46, 0x4f, 0x3a, 0x72, 0x6d,
+ 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x72, 0x77,
+ 0x5f, 0x69, 0x6f, 0x76, 0x65, 0x63, 0x5f, 0x63, 0x62, 0x3a, 0x20, 0x57,
+ 0x72, 0x69, 0x74, 0x65, 0x20, 0x69, 0x6f, 0x76, 0x65, 0x63, 0x20, 0x72,
+ 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x20, 0x72, 0x65, 0x63, 0x65, 0x69,
+ 0x76, 0x65, 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x2f, 0x62, 0x6f, 0x6f,
+ 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x31, 0x28,
+ 0x8e, 0xb1, 0xff, 0xa4, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x07, 0x10,
+ 0xe9, 0x06, 0x18, 0xe9, 0x06, 0x20, 0x00, 0x0a, 0xff, 0x80, 0x80, 0x00,
+ 0x38, 0x06, 0x32, 0x0b, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72,
+ 0x61, 0x67, 0x65, 0x42, 0x5a, 0x49, 0x4e, 0x46, 0x4f, 0x3a, 0x72, 0x6d,
+ 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c,
+ 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a,
+ 0x20, 0x43, 0x61, 0x6c, 0x6c, 0x69, 0x6e, 0x67, 0x20, 0x57, 0x72, 0x69,
+ 0x74, 0x65, 0x20, 0x5b, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x3d, 0x30,
+ 0x2c, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x3d, 0x32, 0x30, 0x39, 0x37, 0x31,
+ 0x35, 0x32, 0x5d, 0x66, 0x6f, 0x72, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74,
+ 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x31, 0x21, 0x28,
+ 0xe6, 0xae, 0x92, 0xa5, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x07, 0x10,
+ 0xf1, 0x0b, 0x18, 0xf1, 0x0b, 0x20, 0x00, 0x0a, 0x85, 0x81, 0x80, 0x00,
+ 0x38, 0x04, 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72,
+ 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x59,
+ 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f,
+ 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61,
+ 0x64, 0x3a, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64,
+ 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x31, 0x3a, 0x20, 0x72, 0x65, 0x71, 0x5f,
+ 0x68, 0x3d, 0x30, 0x78, 0x31, 0x35, 0x20, 0x6d, 0x73, 0x67, 0x5f, 0x69,
+ 0x64, 0x3d, 0x33, 0x3a, 0x20, 0x42, 0x79, 0x74, 0x65, 0x73, 0x20, 0x77,
+ 0x72, 0x69, 0x74, 0x74, 0x65, 0x6e, 0x20, 0x3d, 0x20, 0x32, 0x30, 0x39,
+ 0x37, 0x31, 0x35, 0x32, 0x0a, 0x28, 0x96, 0x87, 0xd7, 0xb3, 0xbe, 0xdc,
+ 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, 0xf1, 0x0b, 0x20,
+ 0x00, 0x0a, 0x88, 0x81, 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, 0x65,
+ 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f,
+ 0x72, 0x61, 0x67, 0x65, 0x42, 0x5c, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74,
+ 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+ 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, 0x20, 0x2f, 0x62, 0x6f,
+ 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x31,
+ 0x3a, 0x20, 0x72, 0x65, 0x71, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x31, 0x35,
+ 0x20, 0x6d, 0x73, 0x67, 0x5f, 0x69, 0x64, 0x3d, 0x33, 0x3a, 0x20, 0x53,
+ 0x65, 0x6e, 0x64, 0x20, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+ 0x3a, 0x20, 0x72, 0x65, 0x73, 0x3d, 0x30, 0x20, 0x65, 0x72, 0x72, 0x3d,
+ 0x30, 0x0a, 0x28, 0xda, 0xb3, 0xd8, 0xb3, 0xbe, 0xdc, 0xa5, 0xbc, 0x15,
+ 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, 0xf1, 0x0b, 0x20, 0x00, 0x0a, 0xae,
+ 0x81, 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f,
+ 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67,
+ 0x65, 0x42, 0x81, 0x01, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72,
+ 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74,
+ 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74,
+ 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x31, 0x3a, 0x20,
+ 0x41, 0x62, 0x6f, 0x75, 0x74, 0x20, 0x74, 0x6f, 0x20, 0x62, 0x6c, 0x6f,
+ 0x63, 0x6b, 0x20, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61,
+ 0x67, 0x65, 0x20, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x20, 0x74, 0x68,
+ 0x72, 0x65, 0x61, 0x64, 0x20, 0x28, 0x74, 0x68, 0x5f, 0x69, 0x64, 0x3a,
+ 0x20, 0x34, 0x39, 0x30, 0x31, 0x32, 0x38, 0x37, 0x30, 0x30, 0x36, 0x34,
+ 0x30, 0x29, 0x20, 0x77, 0x61, 0x6b, 0x65, 0x6c, 0x6f, 0x63, 0x6b, 0x20,
+ 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x64, 0x3a, 0x20, 0x31, 0x0a,
+ 0x0a, 0x28, 0xe4, 0xa2, 0xe0, 0xb3, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08,
+ 0x00, 0x10, 0xe9, 0x06, 0x18, 0xf1, 0x0b, 0x20, 0x00, 0x0a, 0xe7, 0x80,
+ 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72,
+ 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65,
+ 0x42, 0x3b, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67,
+ 0x65, 0x5f, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74,
+ 0x5f, 0x63, 0x62, 0x3a, 0x20, 0x63, 0x6c, 0x6e, 0x74, 0x5f, 0x68, 0x3d,
+ 0x30, 0x78, 0x31, 0x35, 0x20, 0x63, 0x6f, 0x6e, 0x6e, 0x5f, 0x68, 0x3d,
+ 0x30, 0x78, 0x37, 0x32, 0x31, 0x64, 0x61, 0x30, 0x37, 0x31, 0x30, 0x30,
+ 0x0a, 0x28, 0xeb, 0xea, 0x8f, 0xb4, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08,
+ 0x00, 0x10, 0xe9, 0x06, 0x18, 0xe9, 0x06, 0x20, 0x00, 0x0a, 0xf6, 0x80,
+ 0x80, 0x00, 0x38, 0x06, 0x32, 0x0b, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74,
+ 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x51, 0x49, 0x4e, 0x46, 0x4f, 0x3a,
+ 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f,
+ 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61,
+ 0x64, 0x3a, 0x20, 0x44, 0x6f, 0x6e, 0x65, 0x20, 0x57, 0x72, 0x69, 0x74,
+ 0x65, 0x20, 0x28, 0x62, 0x79, 0x74, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x32,
+ 0x30, 0x39, 0x37, 0x31, 0x35, 0x32, 0x29, 0x20, 0x66, 0x6f, 0x72, 0x20,
+ 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f,
+ 0x66, 0x73, 0x31, 0x21, 0x28, 0x9e, 0xa9, 0xc5, 0xb3, 0xbe, 0xdc, 0xa5,
+ 0xbc, 0x15, 0x08, 0x07, 0x10, 0xf1, 0x0b, 0x18, 0xf1, 0x0b, 0x20, 0x00,
+ 0x18, 0x8f, 0x4e, 0x0a, 0x70, 0x40, 0xd2, 0x9f, 0x8f, 0xed, 0x90, 0x55,
+ 0xba, 0x02, 0xe0, 0x80, 0x80, 0x00, 0x0a, 0xdb, 0x80, 0x80, 0x00, 0x38,
+ 0x04, 0x32, 0x08, 0x70, 0x65, 0x72, 0x66, 0x65, 0x74, 0x74, 0x6f, 0x42,
+ 0x39, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x5f, 0x6c, 0x6f, 0x67,
+ 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
+ 0x2e, 0x63, 0x63, 0x3a, 0x32, 0x39, 0x31, 0x20, 0x53, 0x65, 0x65, 0x6e,
+ 0x20, 0x31, 0x31, 0x20, 0x41, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x20,
+ 0x6c, 0x6f, 0x67, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x28, 0xf4,
+ 0x9d, 0x96, 0xd4, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xb5,
+ 0x58, 0x18, 0xb5, 0x58, 0x20, 0x00, 0x18, 0x8f, 0x4e, 0x0a, 0x4e, 0x40,
+ 0xfe, 0xf3, 0x83, 0xd1, 0x9e, 0x55, 0xba, 0x02, 0xbe, 0x80, 0x80, 0x00,
+ 0x0a, 0xb9, 0x80, 0x80, 0x00, 0x38, 0x06, 0x32, 0x00, 0x42, 0x1f, 0x74,
+ 0x75, 0x69, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x3a, 0x73, 0x75, 0x69, 0x73,
+ 0x76, 0x63, 0x20, 0x20, 0x65, 0x78, 0x69, 0x73, 0x20, 0x61, 0x6c, 0x72,
+ 0x65, 0x61, 0x64, 0x79, 0x20, 0x30, 0x28, 0x9f, 0x92, 0xdf, 0xdf, 0xcc,
+ 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xd9, 0x06, 0x18, 0xd9, 0x06,
+ 0x20, 0x00, 0x18, 0x8f, 0x4e, 0x0a, 0x1b, 0x40, 0x9b, 0x83, 0xa7, 0xba,
+ 0xba, 0x55, 0xba, 0x02, 0x8b, 0x80, 0x80, 0x00, 0x12, 0x86, 0x80, 0x80,
+ 0x00, 0x08, 0x1a, 0x18, 0x00, 0x10, 0x00, 0x18, 0x8f, 0x4e};
+
+ static const uint8_t kFilterBytecode[]{
+ 0x0b, 0x01, 0x00, 0x0b, 0x60, 0x13, 0x02, 0x19, 0x23, 0x08, 0x2b, 0x09,
+ 0x33, 0x0f, 0x3b, 0x10, 0x41, 0x4b, 0x05, 0x51, 0x5b, 0x12, 0x63, 0x72,
+ 0x69, 0x8b, 0x02, 0x21, 0x93, 0x02, 0x33, 0x9b, 0x02, 0x35, 0xa1, 0x02,
+ 0xab, 0x02, 0x37, 0xb3, 0x02, 0x07, 0xbb, 0x02, 0x3e, 0xc3, 0x02, 0x3d,
+ 0xca, 0x02, 0x02, 0xdb, 0x02, 0x1b, 0xe3, 0x02, 0x11, 0xeb, 0x02, 0x40,
+ 0xf3, 0x02, 0x04, 0xfb, 0x02, 0x41, 0x83, 0x03, 0x34, 0x8b, 0x03, 0x42,
+ 0x91, 0x03, 0x9b, 0x03, 0x43, 0xa3, 0x03, 0x46, 0xab, 0x03, 0x49, 0xb3,
+ 0x03, 0x04, 0xbb, 0x03, 0x07, 0xc3, 0x03, 0x4b, 0xcb, 0x03, 0x4c, 0xd1,
+ 0x03, 0xdb, 0x03, 0x73, 0xe3, 0x03, 0x5f, 0xeb, 0x03, 0x5b, 0xf3, 0x03,
+ 0x4d, 0xfb, 0x03, 0x04, 0x83, 0x04, 0x5d, 0x8b, 0x04, 0x4e, 0x93, 0x04,
+ 0x4f, 0x9b, 0x04, 0x50, 0xa3, 0x04, 0x51, 0xab, 0x04, 0x3c, 0xb3, 0x04,
+ 0x0e, 0xbb, 0x04, 0x04, 0xc3, 0x04, 0x69, 0xcb, 0x04, 0x53, 0xd3, 0x04,
+ 0x3c, 0xdb, 0x04, 0x04, 0xe3, 0x04, 0x56, 0xeb, 0x04, 0x58, 0xf3, 0x04,
+ 0x5a, 0xa3, 0x38, 0x70, 0x00, 0x0b, 0x03, 0x13, 0x04, 0x19, 0x00, 0x0a,
+ 0x03, 0x23, 0x04, 0x29, 0x00, 0x0a, 0x03, 0x00, 0x0b, 0x06, 0x11, 0x00,
+ 0x0a, 0x0a, 0x5b, 0x07, 0x62, 0x03, 0x00, 0x0a, 0x04, 0x00, 0x0a, 0x02,
+ 0x1b, 0x04, 0x00, 0x0b, 0x0a, 0x13, 0x0d, 0x1b, 0x0e, 0x21, 0x2b, 0x0e,
+ 0x00, 0x0a, 0x0d, 0x73, 0x0b, 0x7a, 0x02, 0x00, 0x0a, 0x09, 0x53, 0x0c,
+ 0x00, 0x0a, 0x02, 0x1b, 0x0c, 0x23, 0x0c, 0x2a, 0x04, 0x00, 0x0a, 0x05,
+ 0x00, 0x0a, 0x02, 0x00, 0x0b, 0x07, 0x11, 0x00, 0x0b, 0x0e, 0x13, 0x0e,
+ 0x1b, 0x11, 0x22, 0x02, 0x33, 0x0e, 0x39, 0x43, 0x0e, 0x49, 0x53, 0x0e,
+ 0x00, 0x0a, 0x08, 0x00, 0x0a, 0x03, 0x23, 0x13, 0x2b, 0x15, 0x33, 0x20,
+ 0x42, 0x05, 0x82, 0x01, 0x02, 0xa1, 0x01, 0xab, 0x01, 0x0e, 0xb2, 0x01,
+ 0x02, 0xc3, 0x01, 0x16, 0xcb, 0x01, 0x0e, 0xd3, 0x01, 0x15, 0xdb, 0x01,
+ 0x0e, 0xe3, 0x01, 0x07, 0xeb, 0x01, 0x1e, 0xf2, 0x01, 0x02, 0x83, 0x02,
+ 0x1f, 0x8b, 0x02, 0x07, 0x91, 0x02, 0x9b, 0x02, 0x0e, 0xa1, 0x02, 0xb3,
+ 0x02, 0x04, 0xbb, 0x02, 0x15, 0xc3, 0x02, 0x15, 0xcb, 0x02, 0x04, 0xd1,
+ 0x02, 0xdb, 0x02, 0x15, 0xe2, 0x02, 0x03, 0x00, 0x0a, 0x07, 0x43, 0x14,
+ 0x4a, 0x02, 0x5b, 0x13, 0x63, 0x13, 0x00, 0x0a, 0x02, 0x1b, 0x14, 0x23,
+ 0x14, 0x2a, 0x04, 0x00, 0x09, 0x00, 0x0b, 0x17, 0x12, 0x0c, 0x73, 0x19,
+ 0x7b, 0x1c, 0x83, 0x01, 0x1d, 0x8b, 0x01, 0x1b, 0x00, 0x0b, 0x0d, 0x13,
+ 0x18, 0x00, 0x0a, 0x2e, 0x00, 0x0a, 0x03, 0x23, 0x1a, 0x2b, 0x1a, 0x33,
+ 0x1b, 0x00, 0x0a, 0x09, 0x53, 0x07, 0x00, 0x0a, 0x07, 0x00, 0x09, 0x13,
+ 0x1a, 0x00, 0x0a, 0x03, 0x23, 0x1a, 0x00, 0x0a, 0x03, 0x23, 0x0e, 0x2a,
+ 0x02, 0x00, 0x0a, 0x0a, 0x00, 0x0a, 0x04, 0x32, 0x0a, 0x92, 0x01, 0x02,
+ 0x00, 0x0b, 0x22, 0x13, 0x23, 0x1a, 0x03, 0x33, 0x04, 0x3b, 0x07, 0x42,
+ 0x03, 0x5b, 0x15, 0x62, 0x03, 0x81, 0x01, 0x8b, 0x01, 0x32, 0x92, 0x01,
+ 0x02, 0xa3, 0x01, 0x1b, 0xab, 0x01, 0x15, 0xb2, 0x01, 0x03, 0xcb, 0x01,
+ 0x0d, 0xda, 0x01, 0x05, 0x83, 0x02, 0x15, 0x00, 0x09, 0x21, 0x00, 0x0b,
+ 0x24, 0x12, 0x02, 0x00, 0x0a, 0x04, 0x32, 0x03, 0xa3, 0x06, 0x25, 0xab,
+ 0x06, 0x0d, 0xb3, 0x06, 0x26, 0xbb, 0x06, 0x27, 0xc3, 0x06, 0x1b, 0xcb,
+ 0x06, 0x28, 0xd3, 0x06, 0x07, 0xdb, 0x06, 0x2b, 0xe3, 0x06, 0x07, 0xeb,
+ 0x06, 0x15, 0xf3, 0x06, 0x2a, 0xfb, 0x06, 0x2c, 0x83, 0x07, 0x0e, 0x8b,
+ 0x07, 0x07, 0x93, 0x07, 0x15, 0x9b, 0x07, 0x2f, 0xc1, 0x3e, 0xcb, 0x3e,
+ 0x30, 0xf9, 0xff, 0xff, 0xff, 0x07, 0x00, 0x0a, 0x03, 0x52, 0x02, 0x63,
+ 0x15, 0x6a, 0x02, 0x00, 0x0a, 0x05, 0x33, 0x0e, 0x00, 0x0a, 0x04, 0x32,
+ 0x03, 0x00, 0x0a, 0x02, 0x22, 0x02, 0x33, 0x29, 0x3a, 0x05, 0x6a, 0x0e,
+ 0x00, 0x2a, 0x02, 0x00, 0x0a, 0x02, 0x1b, 0x0e, 0x22, 0x04, 0x00, 0x09,
+ 0x1a, 0x02, 0x00, 0x0a, 0x0d, 0x7b, 0x2d, 0x83, 0x01, 0x2e, 0x8a, 0x01,
+ 0x02, 0x00, 0x0a, 0x02, 0x1b, 0x0e, 0x21, 0x00, 0x0b, 0x0d, 0x11, 0x00,
+ 0x09, 0xa3, 0x06, 0x0e, 0x00, 0x0a, 0x05, 0x33, 0x31, 0x00, 0x0a, 0x0e,
+ 0x00, 0x09, 0x13, 0x0d, 0x19, 0x00, 0x09, 0x13, 0x34, 0x1a, 0x02, 0x00,
+ 0x0a, 0x09, 0x00, 0x0b, 0x36, 0x12, 0x09, 0x5b, 0x0d, 0x00, 0x0a, 0x13,
+ 0x00, 0x0b, 0x0e, 0x13, 0x07, 0x1b, 0x0e, 0x23, 0x11, 0x2b, 0x38, 0x32,
+ 0x02, 0x00, 0x09, 0x13, 0x3b, 0x1a, 0x02, 0x2b, 0x39, 0x32, 0x09, 0x00,
+ 0x0a, 0x03, 0x23, 0x3a, 0x2a, 0x02, 0x00, 0x0b, 0x04, 0x00, 0x0a, 0x06,
+ 0x42, 0x02, 0x00, 0x0a, 0x06, 0x00, 0x0b, 0x07, 0x13, 0x04, 0x00, 0x0b,
+ 0x3f, 0x13, 0x04, 0x00, 0x0a, 0x08, 0x4b, 0x07, 0x00, 0x0b, 0x07, 0x12,
+ 0x02, 0x00, 0x0b, 0x0d, 0x12, 0x02, 0x00, 0x0a, 0x06, 0x3b, 0x0e, 0x42,
+ 0x02, 0x00, 0x0b, 0x44, 0x12, 0x02, 0x00, 0x0b, 0x45, 0x13, 0x45, 0x00,
+ 0x09, 0x13, 0x04, 0x1b, 0x0e, 0x00, 0x0b, 0x47, 0x13, 0x04, 0x19, 0x00,
+ 0x0b, 0x48, 0x13, 0x0d, 0x1a, 0x03, 0x00, 0x0a, 0x03, 0x2a, 0x06, 0x00,
+ 0x0a, 0x05, 0x33, 0x0e, 0x3b, 0x4a, 0x42, 0x08, 0x00, 0x0b, 0x0e, 0x13,
+ 0x0e, 0x1b, 0x0e, 0x00, 0x09, 0x13, 0x1b, 0x23, 0x0e, 0x2a, 0x02, 0x3b,
+ 0x0e, 0x43, 0x0e, 0x4b, 0x11, 0x00, 0x0b, 0x0d, 0x00, 0x0a, 0x08, 0x4b,
+ 0x07, 0x82, 0x01, 0x05, 0x00, 0x0b, 0x0d, 0x13, 0x3c, 0x00, 0x0a, 0x06,
+ 0x82, 0x01, 0x03, 0x9b, 0x01, 0x15, 0x00, 0x0b, 0x0e, 0x00, 0x09, 0x13,
+ 0x52, 0x00, 0x0a, 0x0f, 0x00, 0x0a, 0x02, 0x1b, 0x54, 0x00, 0x09, 0x13,
+ 0x55, 0x1b, 0x07, 0x00, 0x0a, 0x04, 0x2b, 0x07, 0x00, 0x0b, 0x04, 0x13,
+ 0x11, 0x1b, 0x0d, 0x23, 0x57, 0x2b, 0x15, 0x00, 0x0a, 0x0b, 0x00, 0x0b,
+ 0x59, 0x12, 0x02, 0x23, 0x0e, 0x00, 0x0b, 0x07, 0x00, 0x0a, 0x02, 0x1b,
+ 0x0e, 0x00, 0x0a, 0x02, 0x1b, 0x5c, 0x00, 0x09, 0x13, 0x04, 0x00, 0x0a,
+ 0x02, 0x1b, 0x5e, 0x00, 0x0a, 0x02, 0x1b, 0x0e, 0x23, 0x0e, 0x00, 0x0a,
+ 0x02, 0x1b, 0x1b, 0x23, 0x11, 0x29, 0x33, 0x0d, 0x3b, 0x0e, 0x43, 0x3c,
+ 0x00, 0x09, 0x13, 0x61, 0x19, 0x23, 0x57, 0x00, 0x0a, 0x02, 0x1b, 0x0e,
+ 0x23, 0x1b, 0x5b, 0x0e, 0x63, 0x04, 0x6b, 0x0e, 0x73, 0x04, 0x7b, 0x04,
+ 0x83, 0x01, 0x04, 0x8b, 0x01, 0x0d, 0x93, 0x01, 0x04, 0x9b, 0x01, 0x04,
+ 0xa3, 0x01, 0x0d, 0xab, 0x01, 0x15, 0xb3, 0x01, 0x15, 0xbb, 0x01, 0x0e,
+ 0xc3, 0x01, 0x15, 0xcb, 0x01, 0x15, 0xd3, 0x01, 0x15, 0xdb, 0x01, 0x0d,
+ 0xe3, 0x01, 0x3c, 0xeb, 0x01, 0x04, 0xf3, 0x01, 0x3c, 0xfb, 0x01, 0x0d,
+ 0x83, 0x02, 0x3c, 0x8b, 0x02, 0x1b, 0x93, 0x02, 0x3c, 0x9b, 0x02, 0x0d,
+ 0xa3, 0x02, 0x04, 0xab, 0x02, 0x0e, 0xb3, 0x02, 0x0e, 0xbb, 0x02, 0x0e,
+ 0xc3, 0x02, 0x04, 0xcb, 0x02, 0x0d, 0xd3, 0x02, 0x0d, 0xdb, 0x02, 0x07,
+ 0xe3, 0x02, 0x04, 0xeb, 0x02, 0x1b, 0xf3, 0x02, 0x04, 0xfb, 0x02, 0x15,
+ 0x83, 0x03, 0x0e, 0x8b, 0x03, 0x15, 0x93, 0x03, 0x1b, 0x9b, 0x03, 0x15,
+ 0xa3, 0x03, 0x0d, 0xab, 0x03, 0x15, 0xb3, 0x03, 0x15, 0xbb, 0x03, 0x15,
+ 0xc3, 0x03, 0x15, 0xcb, 0x03, 0x15, 0xd3, 0x03, 0x0e, 0xdb, 0x03, 0x0d,
+ 0xe3, 0x03, 0x15, 0xeb, 0x03, 0x15, 0xf3, 0x03, 0x15, 0xfb, 0x03, 0x15,
+ 0x83, 0x04, 0x15, 0x8b, 0x04, 0x04, 0x93, 0x04, 0x0e, 0x9b, 0x04, 0x0d,
+ 0xa3, 0x04, 0x04, 0xab, 0x04, 0x04, 0xb3, 0x04, 0x04, 0xbb, 0x04, 0x0d,
+ 0xc3, 0x04, 0x04, 0xcb, 0x04, 0x04, 0xd3, 0x04, 0x04, 0xdb, 0x04, 0x04,
+ 0xe3, 0x04, 0x0e, 0xeb, 0x04, 0x07, 0xf3, 0x04, 0x07, 0xfb, 0x04, 0x62,
+ 0x83, 0x05, 0x04, 0x8b, 0x05, 0x07, 0x93, 0x05, 0x11, 0x9b, 0x05, 0x0d,
+ 0xa3, 0x05, 0x62, 0xab, 0x05, 0x0e, 0xb3, 0x05, 0x04, 0xbb, 0x05, 0x1b,
+ 0xc3, 0x05, 0x04, 0xcb, 0x05, 0x15, 0xd3, 0x05, 0x15, 0xdb, 0x05, 0x11,
+ 0xe3, 0x05, 0x0e, 0xeb, 0x05, 0x0e, 0xf3, 0x05, 0x1f, 0xfb, 0x05, 0x04,
+ 0x83, 0x06, 0x15, 0x8b, 0x06, 0x0d, 0x93, 0x06, 0x0d, 0x9b, 0x06, 0x0d,
+ 0xa3, 0x06, 0x3c, 0xab, 0x06, 0x3c, 0xb3, 0x06, 0x3c, 0xbb, 0x06, 0x3c,
+ 0xc3, 0x06, 0x07, 0xcb, 0x06, 0x07, 0xd3, 0x06, 0x07, 0xdb, 0x06, 0x15,
+ 0xe3, 0x06, 0x04, 0xeb, 0x06, 0x0e, 0xf3, 0x06, 0x07, 0xfb, 0x06, 0x04,
+ 0x83, 0x07, 0x04, 0x8b, 0x07, 0x04, 0x93, 0x07, 0x0d, 0x9b, 0x07, 0x0d,
+ 0xa3, 0x07, 0x0d, 0xab, 0x07, 0x0d, 0xb3, 0x07, 0x0d, 0xbb, 0x07, 0x0d,
+ 0xc3, 0x07, 0x3c, 0xcb, 0x07, 0x04, 0xd3, 0x07, 0x0d, 0xdb, 0x07, 0x15,
+ 0xe3, 0x07, 0x3c, 0xeb, 0x07, 0x3c, 0xf3, 0x07, 0x1b, 0x83, 0x08, 0x1b,
+ 0x8b, 0x08, 0x3c, 0x93, 0x08, 0x0d, 0x9b, 0x08, 0x0d, 0xa3, 0x08, 0x04,
+ 0xab, 0x08, 0x0e, 0xb3, 0x08, 0x07, 0xbb, 0x08, 0x57, 0xc3, 0x08, 0x07,
+ 0xcb, 0x08, 0x04, 0xd3, 0x08, 0x07, 0xdb, 0x08, 0x11, 0xe3, 0x08, 0x1b,
+ 0xeb, 0x08, 0x34, 0xf3, 0x08, 0x1f, 0xfb, 0x08, 0x0d, 0x83, 0x09, 0x0d,
+ 0x8b, 0x09, 0x3c, 0x93, 0x09, 0x04, 0x9b, 0x09, 0x0e, 0xa3, 0x09, 0x04,
+ 0xab, 0x09, 0x3c, 0xb3, 0x09, 0x04, 0xbb, 0x09, 0x3c, 0xc3, 0x09, 0x3c,
+ 0xcb, 0x09, 0x04, 0xd3, 0x09, 0x1b, 0xdb, 0x09, 0x07, 0xe3, 0x09, 0x0d,
+ 0xeb, 0x09, 0x04, 0xf3, 0x09, 0x04, 0xfb, 0x09, 0x04, 0x83, 0x0a, 0x04,
+ 0x8b, 0x0a, 0x1b, 0x93, 0x0a, 0x1f, 0x9b, 0x0a, 0x11, 0xa3, 0x0a, 0x07,
+ 0xab, 0x0a, 0x07, 0xb3, 0x0a, 0x0d, 0xbb, 0x0a, 0x11, 0xc3, 0x0a, 0x0d,
+ 0xcb, 0x0a, 0x0d, 0xd3, 0x0a, 0x1b, 0xdb, 0x0a, 0x04, 0xe3, 0x0a, 0x1b,
+ 0xeb, 0x0a, 0x0d, 0xf3, 0x0a, 0x3c, 0xfb, 0x0a, 0x0d, 0x83, 0x0b, 0x1b,
+ 0x8b, 0x0b, 0x0d, 0x93, 0x0b, 0x3c, 0x9b, 0x0b, 0x3c, 0xa3, 0x0b, 0x3c,
+ 0xab, 0x0b, 0x07, 0xb3, 0x0b, 0x0d, 0xbb, 0x0b, 0x11, 0xc3, 0x0b, 0x07,
+ 0xcb, 0x0b, 0x0d, 0xd3, 0x0b, 0x0d, 0xdb, 0x0b, 0x04, 0xe3, 0x0b, 0x0d,
+ 0xeb, 0x0b, 0x0d, 0xf3, 0x0b, 0x0e, 0xfb, 0x0b, 0x0e, 0x83, 0x0c, 0x04,
+ 0x8b, 0x0c, 0x0e, 0x93, 0x0c, 0x0e, 0x9b, 0x0c, 0x0e, 0xa3, 0x0c, 0x0d,
+ 0xab, 0x0c, 0x0d, 0xb3, 0x0c, 0x04, 0xbb, 0x0c, 0x07, 0xc3, 0x0c, 0x63,
+ 0xcb, 0x0c, 0x0d, 0xd3, 0x0c, 0x0d, 0xdb, 0x0c, 0x1f, 0xe3, 0x0c, 0x3c,
+ 0xeb, 0x0c, 0x0d, 0xf3, 0x0c, 0x0e, 0xfb, 0x0c, 0x04, 0x83, 0x0d, 0x04,
+ 0x8b, 0x0d, 0x11, 0x93, 0x0d, 0x1f, 0x9b, 0x0d, 0x04, 0xa3, 0x0d, 0x0e,
+ 0xab, 0x0d, 0x0d, 0xb3, 0x0d, 0x0d, 0xbb, 0x0d, 0x04, 0xc3, 0x0d, 0x04,
+ 0xcb, 0x0d, 0x07, 0xd3, 0x0d, 0x04, 0xdb, 0x0d, 0x0d, 0xb3, 0x0e, 0x0d,
+ 0xbb, 0x0e, 0x04, 0xc3, 0x0e, 0x1f, 0xcb, 0x0e, 0x1b, 0xd3, 0x0e, 0x0d,
+ 0xdb, 0x0e, 0x07, 0xe3, 0x0e, 0x07, 0xeb, 0x0e, 0x04, 0xf3, 0x0e, 0x07,
+ 0xfb, 0x0e, 0x07, 0x83, 0x0f, 0x04, 0x8b, 0x0f, 0x0e, 0x93, 0x0f, 0x04,
+ 0x9b, 0x0f, 0x0d, 0xa3, 0x0f, 0x11, 0xab, 0x0f, 0x11, 0xb3, 0x0f, 0x3c,
+ 0xbb, 0x0f, 0x1f, 0xc3, 0x0f, 0x11, 0xcb, 0x0f, 0x04, 0xd3, 0x0f, 0x04,
+ 0xdb, 0x0f, 0x0d, 0xe3, 0x0f, 0x04, 0xeb, 0x0f, 0x3c, 0xf3, 0x0f, 0x0d,
+ 0xfb, 0x0f, 0x11, 0x83, 0x10, 0x0d, 0x8b, 0x10, 0x04, 0x93, 0x10, 0x11,
+ 0x9b, 0x10, 0x0d, 0xa3, 0x10, 0x04, 0xab, 0x10, 0x0d, 0xb3, 0x10, 0x0d,
+ 0xbb, 0x10, 0x04, 0xc3, 0x10, 0x07, 0xcb, 0x10, 0x07, 0xd3, 0x10, 0x04,
+ 0xdb, 0x10, 0x0d, 0xe3, 0x10, 0x0d, 0xeb, 0x10, 0x04, 0xf3, 0x10, 0x3c,
+ 0xfb, 0x10, 0x0d, 0x83, 0x11, 0x04, 0x8b, 0x11, 0x0d, 0x93, 0x11, 0x0e,
+ 0x9b, 0x11, 0x0e, 0xa3, 0x11, 0x0e, 0xab, 0x11, 0x0e, 0xb3, 0x11, 0x0e,
+ 0xbb, 0x11, 0x0e, 0xc3, 0x11, 0x15, 0xcb, 0x11, 0x07, 0xd3, 0x11, 0x0d,
+ 0xdb, 0x11, 0x0d, 0xe3, 0x11, 0x0d, 0xeb, 0x11, 0x3c, 0xf3, 0x11, 0x3c,
+ 0xfb, 0x11, 0x0d, 0x83, 0x12, 0x15, 0x8b, 0x12, 0x07, 0x93, 0x12, 0x07,
+ 0x9b, 0x12, 0x15, 0xa3, 0x12, 0x04, 0xab, 0x12, 0x04, 0xb3, 0x12, 0x07,
+ 0xbb, 0x12, 0x07, 0xc3, 0x12, 0x0e, 0xcb, 0x12, 0x0e, 0xd3, 0x12, 0x0e,
+ 0xdb, 0x12, 0x0d, 0xe3, 0x12, 0x3c, 0xeb, 0x12, 0x0d, 0xf3, 0x12, 0x3c,
+ 0xfb, 0x12, 0x0e, 0x83, 0x13, 0x15, 0x8b, 0x13, 0x15, 0x93, 0x13, 0x15,
+ 0x9b, 0x13, 0x0d, 0xa3, 0x13, 0x1b, 0xab, 0x13, 0x07, 0xb3, 0x13, 0x04,
+ 0xbb, 0x13, 0x04, 0xc3, 0x13, 0x07, 0xcb, 0x13, 0x07, 0xd3, 0x13, 0x04,
+ 0xdb, 0x13, 0x04, 0xe3, 0x13, 0x07, 0xeb, 0x13, 0x07, 0xf3, 0x13, 0x07,
+ 0xfb, 0x13, 0x07, 0x83, 0x14, 0x15, 0x8b, 0x14, 0x15, 0x93, 0x14, 0x0e,
+ 0x9b, 0x14, 0x04, 0xa3, 0x14, 0x04, 0xab, 0x14, 0x3c, 0xb3, 0x14, 0x04,
+ 0xbb, 0x14, 0x64, 0xc3, 0x14, 0x07, 0xcb, 0x14, 0x15, 0xd3, 0x14, 0x0e,
+ 0xdb, 0x14, 0x07, 0xe3, 0x14, 0x0e, 0xeb, 0x14, 0x0d, 0xf3, 0x14, 0x15,
+ 0xfb, 0x14, 0x04, 0x83, 0x15, 0x0e, 0x8b, 0x15, 0x0e, 0x93, 0x15, 0x04,
+ 0x9b, 0x15, 0x66, 0xa3, 0x15, 0x04, 0xab, 0x15, 0x07, 0xb3, 0x15, 0x0e,
+ 0xbb, 0x15, 0x07, 0xc3, 0x15, 0x07, 0xcb, 0x15, 0x07, 0xd3, 0x15, 0x07,
+ 0xdb, 0x15, 0x04, 0xe3, 0x15, 0x3c, 0xeb, 0x15, 0x67, 0xf3, 0x15, 0x07,
+ 0xfb, 0x15, 0x04, 0x83, 0x16, 0x07, 0x8b, 0x16, 0x07, 0x93, 0x16, 0x04,
+ 0x9b, 0x16, 0x11, 0xa3, 0x16, 0x68, 0xab, 0x16, 0x3c, 0xb3, 0x16, 0x07,
+ 0x00, 0x0a, 0x10, 0x00, 0x0a, 0x14, 0x00, 0x09, 0x13, 0x65, 0x00, 0x09,
+ 0x1a, 0x03, 0x00, 0x00, 0x09, 0x22, 0x03, 0x00, 0x0a, 0x0c, 0x00, 0x0b,
+ 0x6a, 0x00, 0x0b, 0x6b, 0x00, 0x0a, 0x03, 0x23, 0x6c, 0x2b, 0x6e, 0x3b,
+ 0x6d, 0x52, 0x02, 0x00, 0x09, 0x13, 0x6d, 0x1b, 0x6c, 0x23, 0x6e, 0x33,
+ 0x6d, 0x43, 0x6f, 0x4b, 0x0e, 0x51, 0x00, 0x0a, 0x07, 0x49, 0x00, 0x09,
+ 0x13, 0x0e, 0x29, 0x00, 0x09, 0x13, 0x66, 0x00, 0x0a, 0x04, 0x2b, 0x71,
+ 0x00, 0x09, 0x13, 0x71, 0x1a, 0x04, 0x00, 0x0b, 0x0e, 0x13, 0x0e, 0x1b,
+ 0x0e, 0x23, 0x07, 0x2b, 0x0e, 0x33, 0x07, 0x3b, 0x0e, 0x83, 0x01, 0x0e,
+ 0x8b, 0x01, 0x0e, 0x93, 0x01, 0x0e, 0x9b, 0x01, 0x11, 0xa3, 0x01, 0x0e,
+ 0xab, 0x01, 0x07, 0xb3, 0x01, 0x0e, 0xbb, 0x01, 0x04, 0xc3, 0x01, 0x07,
+ 0xcb, 0x01, 0x0e, 0xd3, 0x01, 0x0e, 0x00, 0x5b, 0x74, 0x63, 0x75, 0xd1,
+ 0x03, 0x00, 0x59, 0xf9, 0x01, 0xe9, 0x02, 0x00, 0x0b, 0x2d, 0x00, 0x89,
+ 0x8c, 0xb0, 0x80, 0x08};
+
+ MessageFilter filt;
+ ASSERT_TRUE(
+ filt.LoadFilterBytecode(kFilterBytecode, sizeof(kFilterBytecode)));
+
+ // Pass the trace in input splitting it in slices of arbitrary size.
+ std::vector<MessageFilter::InputSlice> input_slices;
+ for (size_t i = 0; i < sizeof(kTraceData);) {
+ std::minstd_rand0 rnd_engine(0);
+ size_t slice_size = rnd_engine() % 4096;
+ slice_size = std::min(slice_size, sizeof(kTraceData) - i);
+ input_slices.emplace_back(
+ MessageFilter::InputSlice{kTraceData + i, slice_size});
+ i += slice_size;
+ }
+
+ auto filtered_data =
+ filt.FilterMessageFragments(input_slices.data(), input_slices.size());
+
+ EXPECT_GT(filtered_data.size, 0u);
+ EXPECT_LE(filtered_data.size, sizeof(kTraceData));
+
+ perfetto::protos::Trace original_trace;
+ ASSERT_TRUE(original_trace.ParseFromArray(kTraceData, sizeof(kTraceData)));
+
+ perfetto::protos::Trace filtered_trace;
+ ASSERT_TRUE(filtered_trace.ParseFromArray(
+ filtered_data.data.get(), static_cast<int>(filtered_data.size)));
+
+ // Check that the re-serialized traces are identical.
+ std::string original_ser = original_trace.SerializeAsString();
+ std::string filter_ser = filtered_trace.SerializeAsString();
+
+ EXPECT_EQ(filtered_trace.packet_size(), original_trace.packet_size());
+
+ // Don't use EXPECT_EQ, the string is too big. If this check fails, the gtest
+ // diffing algorithm will take several minutes to compute the diff.
+ // That would mistakenly look like a CI or timing-related issue as the gtest
+ // would fail due to timeout.
+ // If this check fails, use base::HexDump() to investigate.
+ EXPECT_TRUE(original_ser == filter_ser);
+}
+
+} // namespace
+} // namespace protozero
diff --git a/src/tracing/core/BUILD.gn b/src/tracing/core/BUILD.gn
index 1c042d327..c81c7163a 100644
--- a/src/tracing/core/BUILD.gn
+++ b/src/tracing/core/BUILD.gn
@@ -61,6 +61,7 @@ source_set("service") {
"../../../protos/perfetto/trace/perfetto:zero", # For MetatraceWriter.
"../../android_stats",
"../../base",
+ "../../protozero/filtering:message_filter",
]
sources = [
"metatrace_writer.cc",
@@ -74,8 +75,8 @@ source_set("service") {
]
if (is_android && perfetto_build_with_android) {
deps += [
- "../../android_internal:lazy_library_loader",
"../../android_internal:headers",
+ "../../android_internal:lazy_library_loader",
]
}
}
diff --git a/src/tracing/core/tracing_service_impl.cc b/src/tracing/core/tracing_service_impl.cc
index ea17dd4ff..7084dec91 100644
--- a/src/tracing/core/tracing_service_impl.cc
+++ b/src/tracing/core/tracing_service_impl.cc
@@ -72,6 +72,7 @@
#include "perfetto/tracing/core/tracing_service_capabilities.h"
#include "perfetto/tracing/core/tracing_service_state.h"
#include "src/android_stats/statsd_logging_helper.h"
+#include "src/protozero/filtering/message_filter.h"
#include "src/tracing/core/packet_stream_validator.h"
#include "src/tracing/core/shared_memory_arbiter_impl.h"
#include "src/tracing/core/trace_buffer.h"
@@ -718,6 +719,35 @@ base::Status TracingServiceImpl::EnableTracing(ConsumerEndpointImpl* consumer,
tracing_sessions_.size());
}
+ // If the trace config provides a filter bytecode, setup the filter now.
+ // If the filter loading fails, abort the tracing session rather than running
+ // unfiltered.
+ std::unique_ptr<protozero::MessageFilter> trace_filter;
+ if (cfg.has_trace_filter()) {
+ const auto& filt = cfg.trace_filter();
+ const std::string& bytecode = filt.bytecode();
+ trace_filter.reset(new protozero::MessageFilter());
+ if (!trace_filter->LoadFilterBytecode(bytecode.data(), bytecode.size())) {
+ MaybeLogUploadEvent(
+ cfg, PerfettoStatsdAtom::kTracedEnableTracingInvalidFilter);
+ return PERFETTO_SVC_ERR("Trace filter bytecode invalid, aborting");
+ }
+ // The filter is created using perfetto.protos.Trace as root message
+ // (because that makes it possible to play around with the `proto_filter`
+ // tool on actual traces). Here in the service, however, we deal with
+ // perfetto.protos.TracePacket(s), which are one level down (Trace.packet).
+ // The IPC client (or the write_into_filte logic in here) are responsible
+ // for pre-pending the packet preamble (See GetProtoPreamble() calls), but
+ // the preamble is not there at ReadBuffer time. Hence we change the root of
+ // the filtering to start at the Trace.packet level.
+ uint32_t packet_field_id = TracePacket::kPacketFieldNumber;
+ if (!trace_filter->SetFilterRoot(&packet_field_id, 1)) {
+ MaybeLogUploadEvent(
+ cfg, PerfettoStatsdAtom::kTracedEnableTracingInvalidFilter);
+ return PERFETTO_SVC_ERR("Failed to set filter root.");
+ }
+ }
+
const TracingSessionID tsid = ++last_tracing_session_id_;
TracingSession* tracing_session =
&tracing_sessions_
@@ -725,6 +755,9 @@ base::Status TracingServiceImpl::EnableTracing(ConsumerEndpointImpl* consumer,
std::forward_as_tuple(tsid, consumer, cfg, task_runner_))
.first->second;
+ if (trace_filter)
+ tracing_session->trace_filter = std::move(trace_filter);
+
if (cfg.write_into_file()) {
if (!fd ^ !cfg.output_path().empty()) {
tracing_sessions_.erase(tsid);
@@ -2073,6 +2106,48 @@ bool TracingServiceImpl::ReadBuffers(TracingSessionID tsid,
total_slices += packets[i].slices().size();
}
+ // +-------------------------------------------------------------------------+
+ // | NO MORE CHANGES TO |packets| AFTER THIS POINT. |
+ // +-------------------------------------------------------------------------+
+
+ // If the tracing session specified a filter, run all packets through the
+ // filter and replace them with the filter results.
+ // The process below mantains the cardinality of input packets. Even if an
+ // entire packet is filtered out, we emit a zero-sized TracePacket proto. That
+ // makes debugging and reasoning about the trace stats easier.
+ // This place swaps the contents of each |packets| entry in place.
+ if (tracing_session->trace_filter) {
+ auto& trace_filter = *tracing_session->trace_filter;
+ // The filter root shoud be reset from protos.Trace to protos.TracePacket
+ // by the earlier call to SetFilterRoot() in EnableTracing().
+ PERFETTO_DCHECK(trace_filter.root_msg_index() != 0);
+ std::vector<protozero::MessageFilter::InputSlice> filter_input;
+ for (auto it = packets.begin(); it != packets.end(); ++it) {
+ const auto& packet_slices = it->slices();
+ filter_input.clear();
+ filter_input.resize(packet_slices.size());
+ ++tracing_session->filter_input_packets;
+ tracing_session->filter_input_bytes += it->size();
+ for (size_t i = 0; i < packet_slices.size(); ++i)
+ filter_input[i] = {packet_slices[i].start, packet_slices[i].size};
+ auto filtered_packet = trace_filter.FilterMessageFragments(
+ &filter_input[0], filter_input.size());
+
+ // Replace the packet in-place with the filtered one (unless failed).
+ *it = TracePacket();
+ if (filtered_packet.error) {
+ ++tracing_session->filter_errors;
+ PERFETTO_DLOG("Trace packet filtering failed @ packet %" PRIu64,
+ tracing_session->filter_input_packets);
+ continue;
+ }
+ tracing_session->filter_output_bytes += filtered_packet.size;
+ it->AddSlice(Slice::TakeOwnership(std::move(filtered_packet.data),
+ filtered_packet.size));
+
+ } // for (packet)
+ } // if (trace_filter)
+
// If the caller asked us to write into a file by setting
// |write_into_file| == true in the trace config, drain the packets read
// (if any) into the given file descriptor.
@@ -2924,6 +2999,14 @@ TraceStats TracingServiceImpl::GetTraceStats(TracingSession* tracing_session) {
trace_stats.set_patches_discarded(patches_discarded_);
trace_stats.set_invalid_packets(tracing_session->invalid_packets);
+ if (tracing_session->trace_filter) {
+ auto* filt_stats = trace_stats.mutable_filter_stats();
+ filt_stats->set_input_packets(tracing_session->filter_input_packets);
+ filt_stats->set_input_bytes(tracing_session->filter_input_bytes);
+ filt_stats->set_output_bytes(tracing_session->filter_output_bytes);
+ filt_stats->set_errors(tracing_session->filter_errors);
+ }
+
for (BufferID buf_id : tracing_session->buffers_index) {
TraceBuffer* buf = GetBufferByID(buf_id);
if (!buf) {
diff --git a/src/tracing/core/tracing_service_impl.h b/src/tracing/core/tracing_service_impl.h
index e97ba2a1b..27b172cec 100644
--- a/src/tracing/core/tracing_service_impl.h
+++ b/src/tracing/core/tracing_service_impl.h
@@ -47,6 +47,10 @@
#include "src/android_stats/perfetto_atoms.h"
#include "src/tracing/core/id_allocator.h"
+namespace protozero {
+class MessageFilter;
+}
+
namespace perfetto {
namespace base {
@@ -574,6 +578,13 @@ class TracingServiceImpl : public TracingService {
// Periodic task for snapshotting service events (e.g. clocks, sync markers
// etc)
base::PeriodicTask snapshot_periodic_task;
+
+ // When non-NULL the packets should be post-processed using the filter.
+ std::unique_ptr<protozero::MessageFilter> trace_filter;
+ uint64_t filter_input_packets = 0;
+ uint64_t filter_input_bytes = 0;
+ uint64_t filter_output_bytes = 0;
+ uint64_t filter_errors = 0;
};
TracingServiceImpl(const TracingServiceImpl&) = delete;
diff --git a/test/configs/BUILD.gn b/test/configs/BUILD.gn
index 4f77441c8..4c93db1ab 100644
--- a/test/configs/BUILD.gn
+++ b/test/configs/BUILD.gn
@@ -35,11 +35,16 @@ action_foreach("configs") {
"client_api.cfg",
"ftrace.cfg",
"ftrace_largebuffer.cfg",
+ "ftrace_with_filter.cfg",
+ "ftrace_with_ksyms.cfg",
"heapprofd.cfg",
"long_trace.cfg",
+ "mm_events.cfg",
"scheduling.cfg",
"summary.cfg",
"sys_stats.cfg",
+ "thermal.cfg",
+ "traced_perf.cfg",
]
outputs = [ "$root_out_dir/{{source_file_part}}.protobuf" ]
diff --git a/test/configs/ftrace_with_filter.cfg b/test/configs/ftrace_with_filter.cfg
new file mode 100644
index 000000000..54c316171
--- /dev/null
+++ b/test/configs/ftrace_with_filter.cfg
@@ -0,0 +1,36 @@
+buffers {
+ size_kb: 65536
+}
+
+data_sources {
+ config {
+ name: "linux.ftrace"
+ target_buffer: 0
+ ftrace_config {
+ ftrace_events: "sched/sched_process_exec"
+ ftrace_events: "sched/sched_process_exit"
+ ftrace_events: "sched/sched_process_fork"
+ ftrace_events: "sched/sched_process_free"
+ ftrace_events: "sched/sched_process_hang"
+ ftrace_events: "sched/sched_process_wait"
+ ftrace_events: "sched/sched_switch"
+ ftrace_events: "sched/sched_wakeup_new"
+ ftrace_events: "sched/sched_wakeup"
+ ftrace_events: "sched/sched_waking"
+ }
+ }
+}
+
+data_sources {
+ config {
+ name: "linux.process_stats"
+ target_buffer: 0
+ }
+}
+
+trace_filter {
+ # A very minimal bytecode which allows only sched_switch and nothing more.
+ bytecode: "\013\001\000\013\002\101\121\151\321\002\000\011\023\003\031\000\012\002\043\004\000\012\007\000\273\341\337\347\016"
+}
+
+duration_ms: 10000
diff --git a/tools/BUILD.gn b/tools/BUILD.gn
index a8342f934..772f6a239 100644
--- a/tools/BUILD.gn
+++ b/tools/BUILD.gn
@@ -24,6 +24,7 @@ group("tools") {
":copy_protoc",
"compact_reencode",
"ftrace_proto_gen",
+ "proto_filter",
"protoprofile",
]
if (is_linux || is_android) {
diff --git a/tools/install-build-deps b/tools/install-build-deps
index bc7765eec..878f1f683 100755
--- a/tools/install-build-deps
+++ b/tools/install-build-deps
@@ -234,8 +234,8 @@ BUILD_DEPS_HOST = [
# Example traces for regression tests.
Dependency(
'test/data.zip',
- 'https://storage.googleapis.com/perfetto/test-data-20210416-122153.zip',
- '0c3a7b353c1100b783873009949596e75db5ecc4a5cf114f24ccf3f1b08570d3',
+ 'https://storage.googleapis.com/perfetto/test-data-20210513-224349.zip',
+ '3dcc146f4ce38d17fd1f8c4c65af07e7cf7c5c4cb8aa4c7bf73ec3a095d997d1',
'all', 'all',
),
diff --git a/tools/proto_filter/BUILD.gn b/tools/proto_filter/BUILD.gn
new file mode 100644
index 000000000..b2b6997a3
--- /dev/null
+++ b/tools/proto_filter/BUILD.gn
@@ -0,0 +1,29 @@
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("../../gn/perfetto_host_executable.gni")
+
+perfetto_host_executable("proto_filter") {
+ testonly = true
+ deps = [
+ "../../gn:default_deps",
+ "../../gn:protobuf_full",
+ "../../src/base",
+ "../../src/protozero",
+ "../../src/protozero/filtering:bytecode_generator",
+ "../../src/protozero/filtering:filter_util",
+ "../../src/protozero/filtering:message_filter",
+ ]
+ sources = [ "proto_filter.cc" ]
+}
diff --git a/tools/proto_filter/proto_filter.cc b/tools/proto_filter/proto_filter.cc
new file mode 100644
index 000000000..e7443cb99
--- /dev/null
+++ b/tools/proto_filter/proto_filter.cc
@@ -0,0 +1,294 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "perfetto/ext/base/file_utils.h"
+#include "perfetto/ext/base/getopt.h"
+#include "perfetto/ext/base/scoped_file.h"
+#include "perfetto/ext/base/string_utils.h"
+#include "perfetto/ext/base/version.h"
+#include "src/protozero/filtering/filter_util.h"
+#include "src/protozero/filtering/message_filter.h"
+
+namespace perfetto {
+namespace proto_filter {
+namespace {
+
+const char kUsage[] =
+ R"(Usage: proto_filter [-s schema_in] [-i message in] [-o message out] [-f filter in] [-F filter out] [-T filter_oct_out] [-d --dedupe] [-I proto include path] [-r root message]
+
+-s --schema-in: Path to the root .proto file. Required for most operations
+-I --proto_path: Extra include directory for proto includes. If omitted assumed CWD.
+-r --root_message: Fully qualified name for the root proto message (e.g. perfetto.protos.Trace)
+ If omitted the first message defined in the schema will be used.
+-i --msg_in: Path of a binary-encoded proto message which will be filtered.
+-o --msg_out: Path of the binary-encoded filtered proto message written in output.
+-f --filter_in: Path of a filter bytecode file previously generated by this tool.
+-F --filter_out: Path of the filter bytecode file generated from the --schema-in definition.
+-T --filter_oct_out: Like --filter_out, but emits a octal-escaped C string suitable for .pbtx.
+-d --dedupe: Minimize filter size by deduping leaf messages with same field ids.
+
+Example usage:
+
+# Convert a .proto schema file into a diff-friendly list of messages/fields>
+
+ proto_filter -r perfetto.protos.Trace -s protos/perfetto/trace/trace.proto
+
+# Generate the filter bytecode from a .proto schema
+
+ proto_filter -r perfetto.protos.Trace -s protos/perfetto/trace/trace.proto \
+ -F /tmp/bytecode [--dedupe]
+
+# List the used/filtered fields from a trace file
+
+ proto_filter -r perfetto.protos.Trace -s protos/perfetto/trace/trace.proto \
+ -i test/data/example_android_trace_30s.pb -f /tmp/bytecode
+
+# Filter a trace using a filter bytecode
+
+ proto_filter -i test/data/example_android_trace_30s.pb -f /tmp/bytecode \
+ -o /tmp/filtered_trace
+)";
+
+int Main(int argc, char** argv) {
+ static const option long_options[] = {
+ {"help", no_argument, nullptr, 'h'},
+ {"version", no_argument, nullptr, 'v'},
+ {"dedupe", no_argument, nullptr, 'd'},
+ {"proto_path", no_argument, nullptr, 'I'},
+ {"schema_in", no_argument, nullptr, 's'},
+ {"root_message", no_argument, nullptr, 'r'},
+ {"msg_in", no_argument, nullptr, 'i'},
+ {"msg_out", no_argument, nullptr, 'o'},
+ {"filter_in", no_argument, nullptr, 'f'},
+ {"filter_out", no_argument, nullptr, 'F'},
+ {"filter_oct_out", no_argument, nullptr, 'T'},
+ {nullptr, 0, nullptr, 0}};
+
+ std::string msg_in;
+ std::string msg_out;
+ std::string filter_in;
+ std::string schema_in;
+ std::string filter_out;
+ std::string filter_oct_out;
+ std::string proto_path;
+ std::string root_message_arg;
+ bool dedupe = false;
+
+ for (;;) {
+ int option =
+ getopt_long(argc, argv, "hvdI:s:r:i:o:f:F:T:", long_options, nullptr);
+
+ if (option == -1)
+ break; // EOF.
+
+ if (option == 'v') {
+ printf("%s\n", base::GetVersionString());
+ exit(0);
+ }
+
+ if (option == 'd') {
+ dedupe = true;
+ continue;
+ }
+
+ if (option == 'I') {
+ proto_path = optarg;
+ continue;
+ }
+
+ if (option == 's') {
+ schema_in = optarg;
+ continue;
+ }
+
+ if (option == 'r') {
+ root_message_arg = optarg;
+ continue;
+ }
+
+ if (option == 'i') {
+ msg_in = optarg;
+ continue;
+ }
+
+ if (option == 'o') {
+ msg_out = optarg;
+ continue;
+ }
+
+ if (option == 'f') {
+ filter_in = optarg;
+ continue;
+ }
+
+ if (option == 'F') {
+ filter_out = optarg;
+ continue;
+ }
+
+ if (option == 'T') {
+ filter_oct_out = optarg;
+ continue;
+ }
+
+ if (option == 'h') {
+ fprintf(stdout, kUsage);
+ exit(0);
+ }
+
+ fprintf(stderr, kUsage);
+ exit(1);
+ }
+
+ if (msg_in.empty() && filter_in.empty() && schema_in.empty()) {
+ fprintf(stderr, kUsage);
+ return 1;
+ }
+
+ std::string msg_in_data;
+ if (!msg_in.empty()) {
+ PERFETTO_LOG("Loading proto-encoded message from %s", msg_in.c_str());
+ if (!base::ReadFile(msg_in, &msg_in_data)) {
+ PERFETTO_ELOG("Could not open message file %s", msg_in.c_str());
+ return 1;
+ }
+ }
+
+ protozero::FilterUtil filter;
+ if (!schema_in.empty()) {
+ PERFETTO_LOG("Loading proto schema from %s", schema_in.c_str());
+ if (!filter.LoadMessageDefinition(schema_in, root_message_arg,
+ proto_path)) {
+ PERFETTO_ELOG("Failed to parse proto schema from %s", schema_in.c_str());
+ return 1;
+ }
+ if (dedupe)
+ filter.Dedupe();
+ }
+
+ protozero::MessageFilter msg_filter;
+ std::string filter_data;
+ std::string filter_data_src;
+ if (!filter_in.empty()) {
+ PERFETTO_LOG("Loading filter bytecode from %s", filter_in.c_str());
+ if (!base::ReadFile(filter_in, &filter_data)) {
+ PERFETTO_ELOG("Could not open filter file %s", filter_in.c_str());
+ return 1;
+ }
+ filter_data_src = filter_in;
+ } else if (!schema_in.empty()) {
+ PERFETTO_LOG("Generating filter bytecode from %s", schema_in.c_str());
+ filter_data = filter.GenerateFilterBytecode();
+ filter_data_src = schema_in;
+ }
+
+ if (!filter_data.empty()) {
+ const uint8_t* data = reinterpret_cast<const uint8_t*>(filter_data.data());
+ if (!msg_filter.LoadFilterBytecode(data, filter_data.size())) {
+ PERFETTO_ELOG("Failed to parse filter bytecode from %s",
+ filter_data_src.c_str());
+ return 1;
+ }
+ }
+
+ // Write the filter bytecode in output.
+ if (!filter_out.empty()) {
+ auto fd = base::OpenFile(filter_out, O_WRONLY | O_TRUNC | O_CREAT, 0644);
+ if (!fd) {
+ PERFETTO_ELOG("Could not open filter out path %s", filter_out.c_str());
+ return 1;
+ }
+ PERFETTO_LOG("Writing filter bytecode (%zu bytes) into %s",
+ filter_data.size(), filter_out.c_str());
+ base::WriteAll(*fd, filter_data.data(), filter_data.size());
+ }
+
+ if (!filter_oct_out.empty()) {
+ auto fd =
+ base::OpenFile(filter_oct_out, O_WRONLY | O_TRUNC | O_CREAT, 0644);
+ if (!fd) {
+ PERFETTO_ELOG("Could not open filter out path %s",
+ filter_oct_out.c_str());
+ return 1;
+ }
+ std::string oct_str;
+ oct_str.reserve(filter_data.size() * 4 + 64);
+ oct_str.append("trace_filter{\n bytecode: \"");
+ for (char c : filter_data) {
+ uint8_t octect = static_cast<uint8_t>(c);
+ char buf[5]{'\\', '0', '0', '0', 0};
+ for (uint8_t i = 0; i < 3; ++i) {
+ buf[3 - i] = static_cast<char>('0' + static_cast<uint8_t>(octect) % 8);
+ octect /= 8;
+ }
+ oct_str.append(buf);
+ }
+ oct_str.append("\"\n}\n");
+ PERFETTO_LOG("Writing filter bytecode (%zu bytes) into %s", oct_str.size(),
+ filter_oct_out.c_str());
+ base::WriteAll(*fd, oct_str.data(), oct_str.size());
+ }
+
+ // Apply the filter to the input message (if any).
+ std::vector<uint8_t> msg_filtered_data;
+ if (!msg_in.empty()) {
+ PERFETTO_LOG("Applying filter %s to proto message %s",
+ filter_data_src.c_str(), msg_in.c_str());
+ msg_filter.enable_field_usage_tracking(true);
+ auto res = msg_filter.FilterMessage(msg_in_data.data(), msg_in_data.size());
+ if (res.error)
+ PERFETTO_FATAL("Filtering failed");
+ msg_filtered_data.insert(msg_filtered_data.end(), res.data.get(),
+ res.data.get() + res.size);
+ }
+
+ // Write out the filtered message.
+ if (!msg_out.empty()) {
+ PERFETTO_LOG("Writing filtered proto bytes (%zu bytes) into %s",
+ msg_filtered_data.size(), msg_out.c_str());
+ auto fd = base::OpenFile(msg_out, O_WRONLY | O_CREAT, 0644);
+ base::WriteAll(*fd, msg_filtered_data.data(), msg_filtered_data.size());
+ }
+
+ if (!msg_in.empty()) {
+ const auto& field_usage_map = msg_filter.field_usage();
+ for (const auto& it : field_usage_map) {
+ const std::string& field_path_varint = it.first;
+ int32_t num_occurrences = it.second;
+ std::string path_str = filter.LookupField(field_path_varint);
+ printf("%-100s %s %d\n", path_str.c_str(),
+ num_occurrences < 0 ? "DROP" : "PASS", std::abs(num_occurrences));
+ }
+ } else if (!schema_in.empty()) {
+ filter.PrintAsText();
+ }
+
+ if ((!filter_out.empty() || !filter_oct_out.empty()) && !dedupe) {
+ PERFETTO_ELOG(
+ "Warning: looks like you are generating a filter without --dedupe. For "
+ "production use cases, --dedupe can make the output bytecode "
+ "significantly smaller.");
+ }
+ return 0;
+}
+
+} // namespace
+} // namespace proto_filter
+} // namespace perfetto
+
+int main(int argc, char** argv) {
+ return perfetto::proto_filter::Main(argc, argv);
+}
diff --git a/tools/run_android_test b/tools/run_android_test
index 64cc560cf..5f3895171 100755
--- a/tools/run_android_test
+++ b/tools/run_android_test
@@ -143,8 +143,10 @@ def Main():
AdbCall('root')
AdbCall('wait-for-device')
- target_dir = '/data/local/tmp/' + args.test_name
- AdbCall('shell', 'rm -rf "%s"; mkdir -p "%s"' % (2 * (target_dir,)))
+ target_dir = '/data/local/tmp/perfetto_tests'
+ if not args.no_cleanup:
+ AdbCall('shell', 'rm -rf "%s"' % target_dir)
+ AdbCall('shell', 'mkdir -p "%s"' % target_dir)
# Some tests require the trace directory to exist, while true for android
# devices in general some emulators might not have it set up. So we check to
# see if it exists, and if not create it.
diff --git a/tools/test_data.txt b/tools/test_data.txt
index abcb65d30..55ee848de 100644
--- a/tools/test_data.txt
+++ b/tools/test_data.txt
@@ -1,5 +1,8 @@
# List of test deps that should be pushed on the device. Paths are relative
# to the root.
-src/traced/probes/ftrace/test/data/
src/traced/probes/filesystem/testdata/
+src/traced/probes/ftrace/test/data/
+test/data/android_log_ring_buffer_mode.pb
+test/data/example_android_trace_30s.pb
+test/data/full_trace_filter.bytecode
test/data/kallsyms.txt