diff options
author | Primiano Tucci <primiano@google.com> | 2021-06-10 12:15:42 +0000 |
---|---|---|
committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2021-06-10 12:15:42 +0000 |
commit | faba89b6c475a3bc5e3e1591610b47881e573681 (patch) | |
tree | 8482ff828baf9ffa89a544a6b6057523a3b4e75f | |
parent | e1119d64130742b8126c39ba2a955179330a9527 (diff) | |
parent | 633cf4512698f868f03d3736c30f7c4c70eb1bd0 (diff) | |
download | perfetto-faba89b6c475a3bc5e3e1591610b47881e573681.tar.gz |
Cherry-pick proto filter changes on sc-dev am: 633cf45126
Original change: https://googleplex-android-review.googlesource.com/c/platform/external/perfetto/+/14844417
Change-Id: I5d6b1d719097fb9f294339fa49fe7a43a884e6ed
44 files changed, 3072 insertions, 437 deletions
diff --git a/Android.bp b/Android.bp index 7f8b8e072..090910dbd 100644 --- a/Android.bp +++ b/Android.bp @@ -47,8 +47,6 @@ cc_binary { ":perfetto_protos_perfetto_config_process_stats_zero_gen", ":perfetto_protos_perfetto_config_profiling_cpp_gen", ":perfetto_protos_perfetto_config_profiling_zero_gen", - ":perfetto_protos_perfetto_config_proto_filter_cpp_gen", - ":perfetto_protos_perfetto_config_proto_filter_zero_gen", ":perfetto_protos_perfetto_config_sys_stats_cpp_gen", ":perfetto_protos_perfetto_config_sys_stats_zero_gen", ":perfetto_protos_perfetto_config_track_event_cpp_gen", @@ -125,8 +123,6 @@ cc_binary { "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_cpp_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_cpp_gen_headers", @@ -303,8 +299,6 @@ cc_library_shared { ":perfetto_protos_perfetto_config_process_stats_zero_gen", ":perfetto_protos_perfetto_config_profiling_cpp_gen", ":perfetto_protos_perfetto_config_profiling_zero_gen", - ":perfetto_protos_perfetto_config_proto_filter_cpp_gen", - ":perfetto_protos_perfetto_config_proto_filter_zero_gen", ":perfetto_protos_perfetto_config_sys_stats_cpp_gen", ":perfetto_protos_perfetto_config_sys_stats_zero_gen", ":perfetto_protos_perfetto_config_track_event_cpp_gen", @@ -388,8 +382,6 @@ cc_library_shared { "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_cpp_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_cpp_gen_headers", @@ -490,8 +482,6 @@ cc_library_shared { ":perfetto_protos_perfetto_config_process_stats_zero_gen", ":perfetto_protos_perfetto_config_profiling_cpp_gen", ":perfetto_protos_perfetto_config_profiling_zero_gen", - ":perfetto_protos_perfetto_config_proto_filter_cpp_gen", - ":perfetto_protos_perfetto_config_proto_filter_zero_gen", ":perfetto_protos_perfetto_config_sys_stats_cpp_gen", ":perfetto_protos_perfetto_config_sys_stats_zero_gen", ":perfetto_protos_perfetto_config_track_event_cpp_gen", @@ -526,6 +516,9 @@ cc_library_shared { ":perfetto_src_ipc_common", ":perfetto_src_ipc_host", ":perfetto_src_kallsyms_kallsyms", + ":perfetto_src_protozero_filtering_bytecode_common", + ":perfetto_src_protozero_filtering_bytecode_parser", + ":perfetto_src_protozero_filtering_message_filter", ":perfetto_src_protozero_protozero", ":perfetto_src_traced_probes_android_log_android_log", ":perfetto_src_traced_probes_common_common", @@ -579,8 +572,6 @@ cc_library_shared { "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_cpp_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_cpp_gen_headers", @@ -698,8 +689,6 @@ cc_library_static { ":perfetto_protos_perfetto_config_process_stats_zero_gen", ":perfetto_protos_perfetto_config_profiling_cpp_gen", ":perfetto_protos_perfetto_config_profiling_zero_gen", - ":perfetto_protos_perfetto_config_proto_filter_cpp_gen", - ":perfetto_protos_perfetto_config_proto_filter_zero_gen", ":perfetto_protos_perfetto_config_sys_stats_cpp_gen", ":perfetto_protos_perfetto_config_sys_stats_zero_gen", ":perfetto_protos_perfetto_config_track_event_cpp_gen", @@ -733,6 +722,9 @@ cc_library_static { ":perfetto_src_ipc_client", ":perfetto_src_ipc_common", ":perfetto_src_ipc_host", + ":perfetto_src_protozero_filtering_bytecode_common", + ":perfetto_src_protozero_filtering_bytecode_parser", + ":perfetto_src_protozero_filtering_message_filter", ":perfetto_src_protozero_protozero", ":perfetto_src_tracing_client_api_without_backends", ":perfetto_src_tracing_common", @@ -773,8 +765,6 @@ cc_library_static { "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_cpp_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_cpp_gen_headers", @@ -821,8 +811,6 @@ cc_library_static { "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_cpp_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_cpp_gen_headers", @@ -897,8 +885,6 @@ cc_binary { ":perfetto_protos_perfetto_config_process_stats_zero_gen", ":perfetto_protos_perfetto_config_profiling_cpp_gen", ":perfetto_protos_perfetto_config_profiling_zero_gen", - ":perfetto_protos_perfetto_config_proto_filter_cpp_gen", - ":perfetto_protos_perfetto_config_proto_filter_zero_gen", ":perfetto_protos_perfetto_config_sys_stats_cpp_gen", ":perfetto_protos_perfetto_config_sys_stats_zero_gen", ":perfetto_protos_perfetto_config_track_event_cpp_gen", @@ -966,8 +952,6 @@ cc_binary { "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_cpp_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_cpp_gen_headers", @@ -1040,8 +1024,6 @@ cc_library_static { ":perfetto_protos_perfetto_config_process_stats_zero_gen", ":perfetto_protos_perfetto_config_profiling_cpp_gen", ":perfetto_protos_perfetto_config_profiling_zero_gen", - ":perfetto_protos_perfetto_config_proto_filter_cpp_gen", - ":perfetto_protos_perfetto_config_proto_filter_zero_gen", ":perfetto_protos_perfetto_config_sys_stats_cpp_gen", ":perfetto_protos_perfetto_config_sys_stats_zero_gen", ":perfetto_protos_perfetto_config_track_event_cpp_gen", @@ -1092,6 +1074,9 @@ cc_library_static { ":perfetto_src_ipc_host", ":perfetto_src_ipc_perfetto_ipc", ":perfetto_src_kallsyms_kallsyms", + ":perfetto_src_protozero_filtering_bytecode_common", + ":perfetto_src_protozero_filtering_bytecode_parser", + ":perfetto_src_protozero_filtering_message_filter", ":perfetto_src_protozero_protozero", ":perfetto_src_traced_probes_android_log_android_log", ":perfetto_src_traced_probes_common_common", @@ -1152,8 +1137,6 @@ cc_library_static { "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_cpp_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_cpp_gen_headers", @@ -1214,8 +1197,6 @@ cc_library_static { "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_cpp_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_cpp_gen_headers", @@ -1299,8 +1280,6 @@ cc_library_static { ":perfetto_protos_perfetto_config_process_stats_zero_gen", ":perfetto_protos_perfetto_config_profiling_cpp_gen", ":perfetto_protos_perfetto_config_profiling_zero_gen", - ":perfetto_protos_perfetto_config_proto_filter_cpp_gen", - ":perfetto_protos_perfetto_config_proto_filter_zero_gen", ":perfetto_protos_perfetto_config_sys_stats_cpp_gen", ":perfetto_protos_perfetto_config_sys_stats_zero_gen", ":perfetto_protos_perfetto_config_track_event_cpp_gen", @@ -1351,6 +1330,9 @@ cc_library_static { ":perfetto_src_ipc_host", ":perfetto_src_ipc_perfetto_ipc", ":perfetto_src_kallsyms_kallsyms", + ":perfetto_src_protozero_filtering_bytecode_common", + ":perfetto_src_protozero_filtering_bytecode_parser", + ":perfetto_src_protozero_filtering_message_filter", ":perfetto_src_protozero_protozero", ":perfetto_src_traced_probes_android_log_android_log", ":perfetto_src_traced_probes_common_common", @@ -1397,8 +1379,6 @@ cc_library_static { "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_cpp_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_cpp_gen_headers", @@ -1459,8 +1439,6 @@ cc_library_static { "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_cpp_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_cpp_gen_headers", @@ -1696,8 +1674,6 @@ cc_test { ":perfetto_protos_perfetto_config_process_stats_zero_gen", ":perfetto_protos_perfetto_config_profiling_cpp_gen", ":perfetto_protos_perfetto_config_profiling_zero_gen", - ":perfetto_protos_perfetto_config_proto_filter_cpp_gen", - ":perfetto_protos_perfetto_config_proto_filter_zero_gen", ":perfetto_protos_perfetto_config_sys_stats_cpp_gen", ":perfetto_protos_perfetto_config_sys_stats_zero_gen", ":perfetto_protos_perfetto_config_track_event_cpp_gen", @@ -1763,6 +1739,9 @@ cc_test { ":perfetto_src_profiling_memory_ring_buffer", ":perfetto_src_profiling_memory_scoped_spinlock", ":perfetto_src_profiling_memory_wire_protocol", + ":perfetto_src_protozero_filtering_bytecode_common", + ":perfetto_src_protozero_filtering_bytecode_parser", + ":perfetto_src_protozero_filtering_message_filter", ":perfetto_src_protozero_protozero", ":perfetto_src_trace_processor_analysis_analysis", ":perfetto_src_trace_processor_containers_containers", @@ -1857,8 +1836,6 @@ cc_test { "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_cpp_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_cpp_gen_headers", @@ -2419,7 +2396,6 @@ genrule { "protos/perfetto/config/profiling/heapprofd_config.proto", "protos/perfetto/config/profiling/java_hprof_config.proto", "protos/perfetto/config/profiling/perf_event_config.proto", - "protos/perfetto/config/proto_filter.proto", "protos/perfetto/config/stress_test_config.proto", "protos/perfetto/config/sys_stats/sys_stats_config.proto", "protos/perfetto/config/test_config.proto", @@ -3282,112 +3258,6 @@ genrule { ], } -// GN: //protos/perfetto/config:proto_filter_cpp -genrule { - name: "perfetto_protos_perfetto_config_proto_filter_cpp_gen", - srcs: [ - "protos/perfetto/config/proto_filter.proto", - ], - tools: [ - "aprotoc", - "perfetto_src_protozero_protoc_plugin_cppgen_plugin", - ], - cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --plugin=protoc-gen-plugin=$(location perfetto_src_protozero_protoc_plugin_cppgen_plugin) --plugin_out=wrapper_namespace=gen:$(genDir)/external/perfetto/ $(in)", - out: [ - "external/perfetto/protos/perfetto/config/proto_filter.gen.cc", - ], -} - -// GN: //protos/perfetto/config:proto_filter_cpp -genrule { - name: "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - srcs: [ - "protos/perfetto/config/proto_filter.proto", - ], - tools: [ - "aprotoc", - "perfetto_src_protozero_protoc_plugin_cppgen_plugin", - ], - cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --plugin=protoc-gen-plugin=$(location perfetto_src_protozero_protoc_plugin_cppgen_plugin) --plugin_out=wrapper_namespace=gen:$(genDir)/external/perfetto/ $(in)", - out: [ - "external/perfetto/protos/perfetto/config/proto_filter.gen.h", - ], - export_include_dirs: [ - ".", - "protos", - ], -} - -// GN: //protos/perfetto/config:proto_filter_lite -genrule { - name: "perfetto_protos_perfetto_config_proto_filter_lite_gen", - srcs: [ - "protos/perfetto/config/proto_filter.proto", - ], - tools: [ - "aprotoc", - ], - cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --cpp_out=lite=true:$(genDir)/external/perfetto/ $(in)", - out: [ - "external/perfetto/protos/perfetto/config/proto_filter.pb.cc", - ], -} - -// GN: //protos/perfetto/config:proto_filter_lite -genrule { - name: "perfetto_protos_perfetto_config_proto_filter_lite_gen_headers", - srcs: [ - "protos/perfetto/config/proto_filter.proto", - ], - tools: [ - "aprotoc", - ], - cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --cpp_out=lite=true:$(genDir)/external/perfetto/ $(in)", - out: [ - "external/perfetto/protos/perfetto/config/proto_filter.pb.h", - ], - export_include_dirs: [ - ".", - "protos", - ], -} - -// GN: //protos/perfetto/config:proto_filter_zero -genrule { - name: "perfetto_protos_perfetto_config_proto_filter_zero_gen", - srcs: [ - "protos/perfetto/config/proto_filter.proto", - ], - tools: [ - "aprotoc", - "protozero_plugin", - ], - cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --plugin=protoc-gen-plugin=$(location protozero_plugin) --plugin_out=wrapper_namespace=pbzero:$(genDir)/external/perfetto/ $(in)", - out: [ - "external/perfetto/protos/perfetto/config/proto_filter.pbzero.cc", - ], -} - -// GN: //protos/perfetto/config:proto_filter_zero -genrule { - name: "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", - srcs: [ - "protos/perfetto/config/proto_filter.proto", - ], - tools: [ - "aprotoc", - "protozero_plugin", - ], - cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --plugin=protoc-gen-plugin=$(location protozero_plugin) --plugin_out=wrapper_namespace=pbzero:$(genDir)/external/perfetto/ $(in)", - out: [ - "external/perfetto/protos/perfetto/config/proto_filter.pbzero.h", - ], - export_include_dirs: [ - ".", - "protos", - ], -} - // GN: //protos/perfetto/config/sys_stats:cpp genrule { name: "perfetto_protos_perfetto_config_sys_stats_cpp_gen", @@ -7414,6 +7284,11 @@ filegroup { ], } +// GN: //src/protozero/filtering:bytecode_common +filegroup { + name: "perfetto_src_protozero_filtering_bytecode_common", +} + // GN: //src/protozero/filtering:bytecode_generator filegroup { name: "perfetto_src_protozero_filtering_bytecode_generator", @@ -7430,12 +7305,30 @@ filegroup { ], } +// GN: //src/protozero/filtering:filter_util +filegroup { + name: "perfetto_src_protozero_filtering_filter_util", + srcs: [ + "src/protozero/filtering/filter_util.cc", + ], +} + +// GN: //src/protozero/filtering:message_filter +filegroup { + name: "perfetto_src_protozero_filtering_message_filter", + srcs: [ + "src/protozero/filtering/message_filter.cc", + ], +} + // GN: //src/protozero/filtering:unittests filegroup { name: "perfetto_src_protozero_filtering_unittests", srcs: [ "src/protozero/filtering/filter_bytecode_generator_unittest.cc", "src/protozero/filtering/filter_bytecode_parser_unittest.cc", + "src/protozero/filtering/filter_util_unittest.cc", + "src/protozero/filtering/message_filter_unittest.cc", "src/protozero/filtering/message_tokenizer_unittest.cc", ], } @@ -8922,7 +8815,6 @@ cc_library_static { ":perfetto_protos_perfetto_config_power_lite_gen", ":perfetto_protos_perfetto_config_process_stats_lite_gen", ":perfetto_protos_perfetto_config_profiling_lite_gen", - ":perfetto_protos_perfetto_config_proto_filter_lite_gen", ":perfetto_protos_perfetto_config_sys_stats_lite_gen", ":perfetto_protos_perfetto_config_track_event_lite_gen", ":perfetto_protos_perfetto_trace_android_lite_gen", @@ -8956,7 +8848,6 @@ cc_library_static { "perfetto_protos_perfetto_config_power_lite_gen_headers", "perfetto_protos_perfetto_config_process_stats_lite_gen_headers", "perfetto_protos_perfetto_config_profiling_lite_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_lite_gen_headers", "perfetto_protos_perfetto_config_sys_stats_lite_gen_headers", "perfetto_protos_perfetto_config_track_event_lite_gen_headers", "perfetto_protos_perfetto_trace_android_lite_gen_headers", @@ -8986,7 +8877,6 @@ cc_library_static { "perfetto_protos_perfetto_config_power_lite_gen_headers", "perfetto_protos_perfetto_config_process_stats_lite_gen_headers", "perfetto_protos_perfetto_config_profiling_lite_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_lite_gen_headers", "perfetto_protos_perfetto_config_sys_stats_lite_gen_headers", "perfetto_protos_perfetto_config_track_event_lite_gen_headers", "perfetto_protos_perfetto_trace_android_lite_gen_headers", @@ -9071,9 +8961,6 @@ cc_test { ":perfetto_protos_perfetto_config_profiling_cpp_gen", ":perfetto_protos_perfetto_config_profiling_lite_gen", ":perfetto_protos_perfetto_config_profiling_zero_gen", - ":perfetto_protos_perfetto_config_proto_filter_cpp_gen", - ":perfetto_protos_perfetto_config_proto_filter_lite_gen", - ":perfetto_protos_perfetto_config_proto_filter_zero_gen", ":perfetto_protos_perfetto_config_sys_stats_cpp_gen", ":perfetto_protos_perfetto_config_sys_stats_lite_gen", ":perfetto_protos_perfetto_config_sys_stats_zero_gen", @@ -9176,8 +9063,11 @@ cc_test { ":perfetto_src_profiling_symbolizer_symbolizer", ":perfetto_src_profiling_symbolizer_unittests", ":perfetto_src_profiling_unittests", + ":perfetto_src_protozero_filtering_bytecode_common", ":perfetto_src_protozero_filtering_bytecode_generator", ":perfetto_src_protozero_filtering_bytecode_parser", + ":perfetto_src_protozero_filtering_filter_util", + ":perfetto_src_protozero_filtering_message_filter", ":perfetto_src_protozero_filtering_unittests", ":perfetto_src_protozero_protozero", ":perfetto_src_protozero_testing_messages_cpp_gen", @@ -9271,6 +9161,7 @@ cc_test { "libbase", "liblog", "libprocinfo", + "libprotobuf-cpp-full", "libprotobuf-cpp-lite", "libsqlite", "libunwindstack", @@ -9314,9 +9205,6 @@ cc_test { "perfetto_protos_perfetto_config_profiling_cpp_gen_headers", "perfetto_protos_perfetto_config_profiling_lite_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_lite_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers", "perfetto_protos_perfetto_config_sys_stats_lite_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", @@ -9459,7 +9347,6 @@ cc_binary { ":perfetto_protos_perfetto_config_power_zero_gen", ":perfetto_protos_perfetto_config_process_stats_zero_gen", ":perfetto_protos_perfetto_config_profiling_zero_gen", - ":perfetto_protos_perfetto_config_proto_filter_zero_gen", ":perfetto_protos_perfetto_config_sys_stats_zero_gen", ":perfetto_protos_perfetto_config_track_event_zero_gen", ":perfetto_protos_perfetto_config_zero_gen", @@ -9519,7 +9406,6 @@ cc_binary { "perfetto_protos_perfetto_config_power_zero_gen_headers", "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_zero_gen_headers", "perfetto_protos_perfetto_config_zero_gen_headers", @@ -9609,7 +9495,6 @@ cc_binary_host { ":perfetto_protos_perfetto_config_power_zero_gen", ":perfetto_protos_perfetto_config_process_stats_zero_gen", ":perfetto_protos_perfetto_config_profiling_zero_gen", - ":perfetto_protos_perfetto_config_proto_filter_zero_gen", ":perfetto_protos_perfetto_config_sys_stats_zero_gen", ":perfetto_protos_perfetto_config_track_event_zero_gen", ":perfetto_protos_perfetto_config_zero_gen", @@ -9676,7 +9561,6 @@ cc_binary_host { "perfetto_protos_perfetto_config_power_zero_gen_headers", "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_zero_gen_headers", "perfetto_protos_perfetto_config_zero_gen_headers", @@ -9768,8 +9652,6 @@ cc_binary { ":perfetto_protos_perfetto_config_process_stats_zero_gen", ":perfetto_protos_perfetto_config_profiling_cpp_gen", ":perfetto_protos_perfetto_config_profiling_zero_gen", - ":perfetto_protos_perfetto_config_proto_filter_cpp_gen", - ":perfetto_protos_perfetto_config_proto_filter_zero_gen", ":perfetto_protos_perfetto_config_sys_stats_cpp_gen", ":perfetto_protos_perfetto_config_sys_stats_zero_gen", ":perfetto_protos_perfetto_config_track_event_cpp_gen", @@ -9816,6 +9698,9 @@ cc_binary { ":perfetto_src_profiling_perf_regs_parsing", ":perfetto_src_profiling_perf_traced_perf_main", ":perfetto_src_profiling_perf_unwinding", + ":perfetto_src_protozero_filtering_bytecode_common", + ":perfetto_src_protozero_filtering_bytecode_parser", + ":perfetto_src_protozero_filtering_message_filter", ":perfetto_src_protozero_protozero", ":perfetto_src_traced_probes_ftrace_ftrace_procfs", ":perfetto_src_traced_probes_packages_list_packages_list_parser", @@ -9855,8 +9740,6 @@ cc_binary { "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_cpp_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_cpp_gen_headers", @@ -9950,8 +9833,6 @@ cc_binary { ":perfetto_protos_perfetto_config_process_stats_zero_gen", ":perfetto_protos_perfetto_config_profiling_cpp_gen", ":perfetto_protos_perfetto_config_profiling_zero_gen", - ":perfetto_protos_perfetto_config_proto_filter_cpp_gen", - ":perfetto_protos_perfetto_config_proto_filter_zero_gen", ":perfetto_protos_perfetto_config_sys_stats_cpp_gen", ":perfetto_protos_perfetto_config_sys_stats_zero_gen", ":perfetto_protos_perfetto_config_track_event_cpp_gen", @@ -10017,8 +9898,6 @@ cc_binary { "perfetto_protos_perfetto_config_process_stats_zero_gen_headers", "perfetto_protos_perfetto_config_profiling_cpp_gen_headers", "perfetto_protos_perfetto_config_profiling_zero_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_cpp_gen_headers", - "perfetto_protos_perfetto_config_proto_filter_zero_gen_headers", "perfetto_protos_perfetto_config_sys_stats_cpp_gen_headers", "perfetto_protos_perfetto_config_sys_stats_zero_gen_headers", "perfetto_protos_perfetto_config_track_event_cpp_gen_headers", @@ -146,7 +146,6 @@ perfetto_cc_binary( ":protos_perfetto_config_power_zero", ":protos_perfetto_config_process_stats_zero", ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_proto_filter_zero", ":protos_perfetto_config_sys_stats_zero", ":protos_perfetto_config_track_event_cpp", ":protos_perfetto_config_track_event_zero", @@ -179,6 +178,9 @@ perfetto_cc_library( ":src_android_stats_android_stats", ":src_android_stats_perfetto_atoms", ":src_kallsyms_kallsyms", + ":src_protozero_filtering_bytecode_common", + ":src_protozero_filtering_bytecode_parser", + ":src_protozero_filtering_message_filter", ":src_traced_probes_android_log_android_log", ":src_traced_probes_common_common", ":src_traced_probes_data_source", @@ -241,8 +243,6 @@ perfetto_cc_library( ":protos_perfetto_config_process_stats_zero", ":protos_perfetto_config_profiling_cpp", ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_proto_filter_cpp", - ":protos_perfetto_config_proto_filter_zero", ":protos_perfetto_config_sys_stats_cpp", ":protos_perfetto_config_sys_stats_zero", ":protos_perfetto_config_track_event_cpp", @@ -776,6 +776,33 @@ filegroup( ], ) +# GN target: //src/protozero/filtering:bytecode_common +filegroup( + name = "src_protozero_filtering_bytecode_common", + srcs = [ + "src/protozero/filtering/filter_bytecode_common.h", + ], +) + +# GN target: //src/protozero/filtering:bytecode_parser +filegroup( + name = "src_protozero_filtering_bytecode_parser", + srcs = [ + "src/protozero/filtering/filter_bytecode_parser.cc", + "src/protozero/filtering/filter_bytecode_parser.h", + ], +) + +# GN target: //src/protozero/filtering:message_filter +filegroup( + name = "src_protozero_filtering_message_filter", + srcs = [ + "src/protozero/filtering/message_filter.cc", + "src/protozero/filtering/message_filter.h", + "src/protozero/filtering/message_tokenizer.h", + ], +) + # GN target: //src/trace_processor/analysis:analysis filegroup( name = "src_trace_processor_analysis_analysis", @@ -1858,7 +1885,6 @@ perfetto_cc_protocpp_library( ":protos_perfetto_config_power_cpp", ":protos_perfetto_config_process_stats_cpp", ":protos_perfetto_config_profiling_cpp", - ":protos_perfetto_config_proto_filter_cpp", ":protos_perfetto_config_protos", ":protos_perfetto_config_sys_stats_cpp", ":protos_perfetto_config_track_event_cpp", @@ -2181,41 +2207,6 @@ perfetto_cc_protozero_library( ], ) -# GN target: //protos/perfetto/config:proto_filter_cpp -perfetto_cc_protocpp_library( - name = "protos_perfetto_config_proto_filter_cpp", - deps = [ - ":protos_perfetto_config_proto_filter_protos", - ], -) - -# GN target: //protos/perfetto/config:proto_filter_lite -perfetto_cc_proto_library( - name = "protos_perfetto_config_proto_filter_lite", - deps = [ - ":protos_perfetto_config_proto_filter_protos", - ], -) - -# GN target: //protos/perfetto/config:proto_filter_zero -perfetto_proto_library( - name = "protos_perfetto_config_proto_filter_protos", - srcs = [ - "protos/perfetto/config/proto_filter.proto", - ], - visibility = [ - PERFETTO_CONFIG.proto_library_visibility, - ], -) - -# GN target: //protos/perfetto/config:proto_filter_zero -perfetto_cc_protozero_library( - name = "protos_perfetto_config_proto_filter_zero", - deps = [ - ":protos_perfetto_config_proto_filter_protos", - ], -) - # GN target: //protos/perfetto/config:zero perfetto_proto_library( name = "protos_perfetto_config_protos", @@ -2238,7 +2229,6 @@ perfetto_proto_library( ":protos_perfetto_config_power_protos", ":protos_perfetto_config_process_stats_protos", ":protos_perfetto_config_profiling_protos", - ":protos_perfetto_config_proto_filter_protos", ":protos_perfetto_config_sys_stats_protos", ":protos_perfetto_config_track_event_protos", ], @@ -2332,7 +2322,6 @@ perfetto_cc_protozero_library( ":protos_perfetto_config_power_zero", ":protos_perfetto_config_process_stats_zero", ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_proto_filter_zero", ":protos_perfetto_config_protos", ":protos_perfetto_config_sys_stats_zero", ":protos_perfetto_config_track_event_zero", @@ -2353,7 +2342,6 @@ perfetto_cc_protocpp_library( ":protos_perfetto_config_power_cpp", ":protos_perfetto_config_process_stats_cpp", ":protos_perfetto_config_profiling_cpp", - ":protos_perfetto_config_proto_filter_cpp", ":protos_perfetto_config_sys_stats_cpp", ":protos_perfetto_config_track_event_cpp", ":protos_perfetto_ipc_protos", @@ -2374,7 +2362,6 @@ perfetto_cc_ipc_library( ":protos_perfetto_config_power_cpp", ":protos_perfetto_config_process_stats_cpp", ":protos_perfetto_config_profiling_cpp", - ":protos_perfetto_config_proto_filter_cpp", ":protos_perfetto_config_sys_stats_cpp", ":protos_perfetto_config_track_event_cpp", ":protos_perfetto_ipc_cpp", @@ -2402,7 +2389,6 @@ perfetto_proto_library( ":protos_perfetto_config_power_protos", ":protos_perfetto_config_process_stats_protos", ":protos_perfetto_config_profiling_protos", - ":protos_perfetto_config_proto_filter_protos", ":protos_perfetto_config_protos", ":protos_perfetto_config_sys_stats_protos", ":protos_perfetto_config_track_event_protos", @@ -2827,7 +2813,6 @@ perfetto_proto_library( ":protos_perfetto_config_power_protos", ":protos_perfetto_config_process_stats_protos", ":protos_perfetto_config_profiling_protos", - ":protos_perfetto_config_proto_filter_protos", ":protos_perfetto_config_protos", ":protos_perfetto_config_sys_stats_protos", ":protos_perfetto_config_track_event_protos", @@ -2847,7 +2832,6 @@ perfetto_cc_protozero_library( ":protos_perfetto_config_power_zero", ":protos_perfetto_config_process_stats_zero", ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_proto_filter_zero", ":protos_perfetto_config_sys_stats_zero", ":protos_perfetto_config_track_event_zero", ":protos_perfetto_config_zero", @@ -2887,7 +2871,6 @@ perfetto_proto_library( ":protos_perfetto_config_power_protos", ":protos_perfetto_config_process_stats_protos", ":protos_perfetto_config_profiling_protos", - ":protos_perfetto_config_proto_filter_protos", ":protos_perfetto_config_protos", ":protos_perfetto_config_sys_stats_protos", ":protos_perfetto_config_track_event_protos", @@ -2921,7 +2904,6 @@ perfetto_cc_protozero_library( ":protos_perfetto_config_power_zero", ":protos_perfetto_config_process_stats_zero", ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_proto_filter_zero", ":protos_perfetto_config_sys_stats_zero", ":protos_perfetto_config_track_event_zero", ":protos_perfetto_config_zero", @@ -3323,6 +3305,9 @@ perfetto_cc_library( srcs = [ ":src_android_stats_android_stats", ":src_android_stats_perfetto_atoms", + ":src_protozero_filtering_bytecode_common", + ":src_protozero_filtering_bytecode_parser", + ":src_protozero_filtering_message_filter", ":src_tracing_client_api_without_backends", ":src_tracing_common", ":src_tracing_core_core", @@ -3371,8 +3356,6 @@ perfetto_cc_library( ":protos_perfetto_config_process_stats_zero", ":protos_perfetto_config_profiling_cpp", ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_proto_filter_cpp", - ":protos_perfetto_config_proto_filter_zero", ":protos_perfetto_config_sys_stats_cpp", ":protos_perfetto_config_sys_stats_zero", ":protos_perfetto_config_track_event_cpp", @@ -3451,8 +3434,6 @@ perfetto_cc_binary( ":protos_perfetto_config_process_stats_zero", ":protos_perfetto_config_profiling_cpp", ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_proto_filter_cpp", - ":protos_perfetto_config_proto_filter_zero", ":protos_perfetto_config_sys_stats_cpp", ":protos_perfetto_config_sys_stats_zero", ":protos_perfetto_config_track_event_cpp", @@ -3530,7 +3511,6 @@ perfetto_cc_library( ":protos_perfetto_config_power_zero", ":protos_perfetto_config_process_stats_zero", ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_proto_filter_zero", ":protos_perfetto_config_sys_stats_zero", ":protos_perfetto_config_track_event_zero", ":protos_perfetto_config_zero", @@ -3622,7 +3602,6 @@ perfetto_cc_binary( ":protos_perfetto_config_power_zero", ":protos_perfetto_config_process_stats_zero", ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_proto_filter_zero", ":protos_perfetto_config_sys_stats_zero", ":protos_perfetto_config_track_event_zero", ":protos_perfetto_config_zero", @@ -3724,7 +3703,6 @@ perfetto_cc_library( ":protos_perfetto_config_power_zero", ":protos_perfetto_config_process_stats_zero", ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_proto_filter_zero", ":protos_perfetto_config_sys_stats_zero", ":protos_perfetto_config_track_event_zero", ":protos_perfetto_config_zero", @@ -3804,7 +3782,6 @@ perfetto_cc_binary( ":protos_perfetto_config_power_zero", ":protos_perfetto_config_process_stats_zero", ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_proto_filter_zero", ":protos_perfetto_config_sys_stats_zero", ":protos_perfetto_config_track_event_zero", ":protos_perfetto_config_zero", diff --git a/gn/BUILD.gn b/gn/BUILD.gn index a297a845b..1f799ade3 100644 --- a/gn/BUILD.gn +++ b/gn/BUILD.gn @@ -195,7 +195,9 @@ group("gtest_main") { protobuf_full_deps_allowlist = [ "../src/ipc/protoc_plugin:*", "../src/protozero/protoc_plugin:*", + "../src/protozero/filtering:filter_util", "../src/trace_processor:trace_processor_shell", + "../src/protozero/filtering:filter_util", "../tools/*", ] diff --git a/gn/perfetto_benchmarks.gni b/gn/perfetto_benchmarks.gni index 2bde3766f..e1540649c 100644 --- a/gn/perfetto_benchmarks.gni +++ b/gn/perfetto_benchmarks.gni @@ -18,6 +18,7 @@ perfetto_benchmarks_targets = [ "gn:default_deps", "src/base:benchmarks", "src/protozero:benchmarks", + "src/protozero/filtering:benchmarks", "src/trace_processor/sqlite:benchmarks", "src/trace_processor/containers:benchmarks", "src/trace_processor/tables:benchmarks", diff --git a/gn/perfetto_fuzzers.gni b/gn/perfetto_fuzzers.gni index 049d5251b..5874f8e14 100644 --- a/gn/perfetto_fuzzers.gni +++ b/gn/perfetto_fuzzers.gni @@ -19,6 +19,7 @@ perfetto_fuzzers_targets = [ "src/ipc:buffered_frame_deserializer_fuzzer", "src/protozero:protozero_decoder_fuzzer", "src/protozero/filtering:protozero_bytecode_parser_fuzzer", + "src/protozero/filtering:protozero_message_filter_fuzzer", "src/tracing/core:packet_stream_validator_fuzzer", "src/trace_processor:trace_processor_fuzzer", "src/traced/probes/ftrace:cpu_reader_fuzzer", diff --git a/include/perfetto/ext/tracing/core/slice.h b/include/perfetto/ext/tracing/core/slice.h index 4ce66fdea..063043bdc 100644 --- a/include/perfetto/ext/tracing/core/slice.h +++ b/include/perfetto/ext/tracing/core/slice.h @@ -44,6 +44,14 @@ struct Slice { return slice; } + static Slice TakeOwnership(std::unique_ptr<uint8_t[]> buf, size_t size) { + Slice slice; + slice.own_data_ = std::move(buf); + slice.start = &slice.own_data_[0]; + slice.size = size; + return slice; + } + uint8_t* own_data() { PERFETTO_DCHECK(own_data_); return own_data_.get(); diff --git a/include/perfetto/protozero/proto_utils.h b/include/perfetto/protozero/proto_utils.h index 841042b17..6b195cb1e 100644 --- a/include/perfetto/protozero/proto_utils.h +++ b/include/perfetto/protozero/proto_utils.h @@ -201,12 +201,20 @@ inline uint8_t* WriteVarInt(T value, uint8_t* target) { // used to backfill fixed-size reservations for the length field using a // non-canonical varint encoding (e.g. \x81\x80\x80\x00 instead of \x01). // See https://github.com/google/protobuf/issues/1530. -// In particular, this is used for nested messages. The size of a nested message -// is not known until all its field have been written. |kMessageLengthFieldSize| -// bytes are reserved to encode the size field and backfilled at the end. -inline void WriteRedundantVarInt(uint32_t value, uint8_t* buf) { - for (size_t i = 0; i < kMessageLengthFieldSize; ++i) { - const uint8_t msb = (i < kMessageLengthFieldSize - 1) ? 0x80 : 0; +// This is used mainly in two cases: +// 1) At trace writing time, when starting a nested messages. The size of a +// nested message is not known until all its field have been written. +// |kMessageLengthFieldSize| bytes are reserved to encode the size field and +// backfilled at the end. +// 2) When rewriting a message at trace filtering time, in protozero/filtering. +// At that point we know only the upper bound of the length (a filtered +// message is <= the original one) and we backfill after the message has been +// filtered. +inline void WriteRedundantVarInt(uint32_t value, + uint8_t* buf, + size_t size = kMessageLengthFieldSize) { + for (size_t i = 0; i < size; ++i) { + const uint8_t msb = (i < size - 1) ? 0x80 : 0; buf[i] = static_cast<uint8_t>(value) | msb; value >>= 7; } diff --git a/protos/perfetto/common/trace_stats.proto b/protos/perfetto/common/trace_stats.proto index e925ad460..c6a9fbfa1 100644 --- a/protos/perfetto/common/trace_stats.proto +++ b/protos/perfetto/common/trace_stats.proto @@ -158,4 +158,13 @@ message TraceStats { // Packets that failed validation of the TrustedPacket. If this is > 0, there // is a bug in the producer. optional uint64 invalid_packets = 10; + + // This is set only when the TraceConfig specifies a TraceFilter. + message FilterStats { + optional uint64 input_packets = 1; + optional uint64 input_bytes = 2; + optional uint64 output_bytes = 3; + optional uint64 errors = 4; + } + optional FilterStats filter_stats = 11; } diff --git a/protos/perfetto/config/BUILD.gn b/protos/perfetto/config/BUILD.gn index 60bcaf6a8..de9c1f542 100644 --- a/protos/perfetto/config/BUILD.gn +++ b/protos/perfetto/config/BUILD.gn @@ -20,7 +20,6 @@ import("../../../gn/proto_library.gni") perfetto_proto_library("@TYPE@") { deps = [ - ":proto_filter_@TYPE@", "../common:@TYPE@", "android:@TYPE@", "ftrace:@TYPE@", @@ -63,7 +62,3 @@ perfetto_proto_library("perfetto_config_descriptor") { generate_descriptor = "perfetto_config.descriptor" sources = [ "perfetto_config.proto" ] } - -perfetto_proto_library("proto_filter_@TYPE@") { - sources = [ "proto_filter.proto" ] -} diff --git a/protos/perfetto/config/perfetto_config.proto b/protos/perfetto/config/perfetto_config.proto index d088e9b84..5b0f2b692 100644 --- a/protos/perfetto/config/perfetto_config.proto +++ b/protos/perfetto/config/perfetto_config.proto @@ -1462,7 +1462,7 @@ message DataSourceConfig { // It contains the general config for the logging buffer(s) and the configs for // all the data source being enabled. // -// Next id: 31. +// Next id: 33. message TraceConfig { message BufferConfig { optional uint32 size_kb = 1; @@ -1857,6 +1857,15 @@ message TraceConfig { // Alternative encoding of trace_uuid as two int64s. optional int64 trace_uuid_msb = 27; optional int64 trace_uuid_lsb = 28; + + // When set applies a post-filter to the trace contents using the filter + // provided. The filter is applied at ReadBuffers() time and works both in the + // case of IPC readback and write_into_file. This filter can be generated + // using `tools/proto_filter -s schema.proto -F filter_out.bytes` or + // `-T filter_out.escaped_string` (for .pbtx). + // Introduced in Android S. See go/trace-filtering for design. + message TraceFilter { optional bytes bytecode = 1; } + optional TraceFilter trace_filter = 32; } // End of protos/perfetto/config/trace_config.proto diff --git a/protos/perfetto/config/proto_filter.proto b/protos/perfetto/config/proto_filter.proto deleted file mode 100644 index deec6935a..000000000 --- a/protos/perfetto/config/proto_filter.proto +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2021 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto2"; - -package perfetto.protos; - -// This proto wraps a bytecode that describes how to filter proto messages. -// For the full specs of the filter bytecode see go/trace-filtering. -// In short: -// - A filter is a sequence of uint32 words. -// - Each word is (immediate value << 3) | (3-bit OPCODE) -// - Each group of words (until END_OF_MESSAGE) defines field filters for a -// message. The first message is the root message where the filtering starts -// (typically perfetto.protos.Trace). -message ProtoFilter { - enum Opcode { - // The immediate value is 0 in this case. - FILTER_OPCODE_END_OF_MESSAGE = 0; - - // The immediate value is the id of the allowed field. - FILTER_OPCODE_SIMPLE_FIELD = 1; - - // The immediate value is the start of the range. The next word (without - // any shifting) is the length of the range. - FILTER_OPCODE_SIMPLE_FIELD_RANGE = 2; - - // The immediate value is the id of the allowed field. The next word - // (without any shifting) is the index of the filter that should be used to - // recurse into the nested message. - FILTER_OPCODE_NESTED_FIELD = 3; - } - repeated uint32 bytecode = 1 [packed = true]; -} diff --git a/protos/perfetto/config/trace_config.proto b/protos/perfetto/config/trace_config.proto index 57677d3fe..dbc1babda 100644 --- a/protos/perfetto/config/trace_config.proto +++ b/protos/perfetto/config/trace_config.proto @@ -26,7 +26,7 @@ package perfetto.protos; // It contains the general config for the logging buffer(s) and the configs for // all the data source being enabled. // -// Next id: 31. +// Next id: 33. message TraceConfig { message BufferConfig { optional uint32 size_kb = 1; @@ -421,4 +421,13 @@ message TraceConfig { // Alternative encoding of trace_uuid as two int64s. optional int64 trace_uuid_msb = 27; optional int64 trace_uuid_lsb = 28; + + // When set applies a post-filter to the trace contents using the filter + // provided. The filter is applied at ReadBuffers() time and works both in the + // case of IPC readback and write_into_file. This filter can be generated + // using `tools/proto_filter -s schema.proto -F filter_out.bytes` or + // `-T filter_out.escaped_string` (for .pbtx). + // Introduced in Android S. See go/trace-filtering for design. + message TraceFilter { optional bytes bytecode = 1; } + optional TraceFilter trace_filter = 32; } diff --git a/protos/perfetto/trace/perfetto_trace.proto b/protos/perfetto/trace/perfetto_trace.proto index 36313affd..402fae891 100644 --- a/protos/perfetto/trace/perfetto_trace.proto +++ b/protos/perfetto/trace/perfetto_trace.proto @@ -1462,7 +1462,7 @@ message DataSourceConfig { // It contains the general config for the logging buffer(s) and the configs for // all the data source being enabled. // -// Next id: 31. +// Next id: 33. message TraceConfig { message BufferConfig { optional uint32 size_kb = 1; @@ -1857,6 +1857,15 @@ message TraceConfig { // Alternative encoding of trace_uuid as two int64s. optional int64 trace_uuid_msb = 27; optional int64 trace_uuid_lsb = 28; + + // When set applies a post-filter to the trace contents using the filter + // provided. The filter is applied at ReadBuffers() time and works both in the + // case of IPC readback and write_into_file. This filter can be generated + // using `tools/proto_filter -s schema.proto -F filter_out.bytes` or + // `-T filter_out.escaped_string` (for .pbtx). + // Introduced in Android S. See go/trace-filtering for design. + message TraceFilter { optional bytes bytecode = 1; } + optional TraceFilter trace_filter = 32; } // End of protos/perfetto/config/trace_config.proto @@ -2003,6 +2012,15 @@ message TraceStats { // Packets that failed validation of the TrustedPacket. If this is > 0, there // is a bug in the producer. optional uint64 invalid_packets = 10; + + // This is set only when the TraceConfig specifies a TraceFilter. + message FilterStats { + optional uint64 input_packets = 1; + optional uint64 input_bytes = 2; + optional uint64 output_bytes = 3; + optional uint64 errors = 4; + } + optional FilterStats filter_stats = 11; } // End of protos/perfetto/common/trace_stats.proto diff --git a/src/android_stats/perfetto_atoms.h b/src/android_stats/perfetto_atoms.h index cf73d5f65..3731c2e76 100644 --- a/src/android_stats/perfetto_atoms.h +++ b/src/android_stats/perfetto_atoms.h @@ -68,6 +68,7 @@ enum class PerfettoStatsdAtom { kTracedEnableTracingOom = 34, kTracedEnableTracingUnknown = 35, kTracedStartTracingInvalidSessionState = 36, + kTracedEnableTracingInvalidFilter = 47, // Checkpoints inside perfetto_cmd after tracing has finished. kOnTracingDisabled = 4, diff --git a/src/perfetto_cmd/pbtxt_to_pb.cc b/src/perfetto_cmd/pbtxt_to_pb.cc index f4f0c00a1..b78d93829 100644 --- a/src/perfetto_cmd/pbtxt_to_pb.cc +++ b/src/perfetto_cmd/pbtxt_to_pb.cc @@ -47,12 +47,20 @@ using protos::gen::FileDescriptorSet; namespace { +constexpr bool IsOct(char c) { + return (c >= '0' && c <= '7'); +} + +constexpr bool IsDigit(char c) { + return (c >= '0' && c <= '9'); +} + constexpr bool IsIdentifierStart(char c) { return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') || c == '_'; } constexpr bool IsIdentifierBody(char c) { - return IsIdentifierStart(c) || isdigit(c); + return IsIdentifierStart(c) || IsDigit(c); } const char* FieldToTypeName(const FieldDescriptorProto* field) { @@ -150,19 +158,22 @@ class ParserDelegate { } void NumericField(Token key, Token value) { - const FieldDescriptorProto* field = FindFieldByName( - key, value, - { - FieldDescriptorProto::TYPE_UINT64, - FieldDescriptorProto::TYPE_UINT32, FieldDescriptorProto::TYPE_INT64, - FieldDescriptorProto::TYPE_SINT64, FieldDescriptorProto::TYPE_INT32, - FieldDescriptorProto::TYPE_SINT32, - FieldDescriptorProto::TYPE_FIXED64, - FieldDescriptorProto::TYPE_SFIXED64, - FieldDescriptorProto::TYPE_FIXED32, - FieldDescriptorProto::TYPE_SFIXED32, - FieldDescriptorProto::TYPE_DOUBLE, FieldDescriptorProto::TYPE_FLOAT, - }); + const FieldDescriptorProto* field = + FindFieldByName(key, value, + { + FieldDescriptorProto::TYPE_UINT64, + FieldDescriptorProto::TYPE_UINT32, + FieldDescriptorProto::TYPE_INT64, + FieldDescriptorProto::TYPE_SINT64, + FieldDescriptorProto::TYPE_INT32, + FieldDescriptorProto::TYPE_SINT32, + FieldDescriptorProto::TYPE_FIXED64, + FieldDescriptorProto::TYPE_SFIXED64, + FieldDescriptorProto::TYPE_FIXED32, + FieldDescriptorProto::TYPE_SFIXED32, + FieldDescriptorProto::TYPE_DOUBLE, + FieldDescriptorProto::TYPE_FLOAT, + }); if (!field) return; const auto& field_type = field->type(); @@ -202,11 +213,12 @@ class ParserDelegate { } void StringField(Token key, Token value) { - const FieldDescriptorProto* field = FindFieldByName( - key, value, - { - FieldDescriptorProto::TYPE_STRING, FieldDescriptorProto::TYPE_BYTES, - }); + const FieldDescriptorProto* field = + FindFieldByName(key, value, + { + FieldDescriptorProto::TYPE_STRING, + FieldDescriptorProto::TYPE_BYTES, + }); if (!field) return; uint32_t field_id = static_cast<uint32_t>(field->number()); @@ -217,15 +229,16 @@ class ParserDelegate { std::unique_ptr<char, base::FreeDeleter> s( static_cast<char*>(malloc(value.size()))); size_t j = 0; + const char* const txt = value.txt.data(); for (size_t i = 0; i < value.size(); i++) { - char c = value.txt.data()[i]; + char c = txt[i]; if (c == '\\') { if (i + 1 >= value.size()) { // This should be caught by the lexer. PERFETTO_FATAL("Escape at end of string."); return; } - char next = value.txt.data()[++i]; + char next = txt[++i]; switch (next) { case '\\': case '\'': @@ -254,6 +267,44 @@ class ParserDelegate { case 'v': s.get()[j++] = '\v'; break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + // Cases 8 and 9 are not really required and are only added for the + // sake of error reporting. + bool oct_err = false; + if (i + 2 >= value.size() || !IsOct(txt[i + 1]) || + !IsOct(txt[i + 2])) { + oct_err = true; + } else { + char buf[4]{next, txt[++i], txt[++i], '\0'}; + auto octval = base::CStringToUInt32(buf, 8); + if (!octval.has_value() || *octval > 0xff) { + oct_err = true; + } else { + s.get()[j++] = static_cast<char>(static_cast<uint8_t>(*octval)); + } + } + if (oct_err) { + AddError(value, + "Malformed string escape in $k in proto $n on '$v'. " + "\\NNN escapes must be exactly three octal digits <= " + "\\377 (0xff).", + std::map<std::string, std::string>{ + {"$k", key.ToStdString()}, + {"$n", descriptor_name()}, + {"$v", value.ToStdString()}, + }); + } + break; + } default: AddError(value, "Unknown string escape in $k in " @@ -273,11 +324,12 @@ class ParserDelegate { } void IdentifierField(Token key, Token value) { - const FieldDescriptorProto* field = FindFieldByName( - key, value, - { - FieldDescriptorProto::TYPE_BOOL, FieldDescriptorProto::TYPE_ENUM, - }); + const FieldDescriptorProto* field = + FindFieldByName(key, value, + { + FieldDescriptorProto::TYPE_BOOL, + FieldDescriptorProto::TYPE_ENUM, + }); if (!field) return; uint32_t field_id = static_cast<uint32_t>(field->number()); @@ -415,7 +467,8 @@ class ParserDelegate { if (!field_descriptor) { AddError(key, "No field named \"$n\" in proto $p", { - {"$n", field_name}, {"$p", descriptor_name()}, + {"$n", field_name}, + {"$p", descriptor_name()}, }); return nullptr; } @@ -549,7 +602,7 @@ void Parse(const std::string& input, ParserDelegate* delegate) { state = kReadingStringValue; continue; } - if (c == '-' || isdigit(c) || c == '.') { + if (c == '-' || IsDigit(c) || c == '.') { state = kReadingNumericValue; continue; } @@ -576,7 +629,7 @@ void Parse(const std::string& input, ParserDelegate* delegate) { delegate->NumericField(key, value); continue; } - if (isdigit(c) || c == '.') + if (IsDigit(c) || c == '.') continue; break; diff --git a/src/perfetto_cmd/pbtxt_to_pb_unittest.cc b/src/perfetto_cmd/pbtxt_to_pb_unittest.cc index 00dab16f0..251c8f07e 100644 --- a/src/perfetto_cmd/pbtxt_to_pb_unittest.cc +++ b/src/perfetto_cmd/pbtxt_to_pb_unittest.cc @@ -403,6 +403,7 @@ data_sources { ftrace_events: "newline\nnewline" ftrace_events: "\"quoted\"" ftrace_events: "\a\b\f\n\r\t\v\\\'\"\?" + ftrace_events: "\0127_\03422.\177" } } } @@ -417,6 +418,7 @@ data_sources { EXPECT_THAT(events, Contains("newline\nnewline")); EXPECT_THAT(events, Contains("\"quoted\"")); EXPECT_THAT(events, Contains("\a\b\f\n\r\t\v\\\'\"\?")); + EXPECT_THAT(events, Contains("\0127_\03422.\177")); } TEST(PbtxtToPb, UnknownField) { diff --git a/src/protozero/filtering/BUILD.gn b/src/protozero/filtering/BUILD.gn index 0ddfc3eff..b438929b4 100644 --- a/src/protozero/filtering/BUILD.gn +++ b/src/protozero/filtering/BUILD.gn @@ -17,15 +17,34 @@ import("../../../gn/perfetto_host_executable.gni") import("../../../gn/proto_library.gni") import("../../../gn/test.gni") +source_set("message_filter") { + sources = [ + "message_filter.cc", + "message_filter.h", + "message_tokenizer.h", + ] + deps = [ + ":bytecode_parser", + "..:protozero", + "../../../gn:default_deps", + "../../base", + ] +} + +source_set("bytecode_common") { + sources = [ "filter_bytecode_common.h" ] + deps = [ "../../../gn:default_deps" ] +} + source_set("bytecode_parser") { sources = [ "filter_bytecode_parser.cc", "filter_bytecode_parser.h", ] deps = [ + ":bytecode_common", "..:protozero", "../../../gn:default_deps", - "../../../protos/perfetto/config:proto_filter_zero", "../../base", ] } @@ -36,9 +55,24 @@ source_set("bytecode_generator") { "filter_bytecode_generator.h", ] deps = [ + ":bytecode_common", + "..:protozero", + "../../../gn:default_deps", + "../../base", + ] +} + +source_set("filter_util") { + testonly = true + sources = [ + "filter_util.cc", + "filter_util.h", + ] + deps = [ + ":bytecode_generator", "..:protozero", "../../../gn:default_deps", - "../../../protos/perfetto/config:proto_filter_zero", + "../../../gn:protobuf_full", "../../base", ] } @@ -46,26 +80,57 @@ source_set("bytecode_generator") { perfetto_unittest_source_set("unittests") { testonly = true deps = [ + ":bytecode_common", ":bytecode_generator", ":bytecode_parser", + ":filter_util", + ":message_filter", "..:protozero", "../../../gn:default_deps", "../../../gn:gtest_and_gmock", - "../../../protos/perfetto/config:proto_filter_zero", + "../../../protos/perfetto/trace:lite", "../../base", "../../base:test_support", ] sources = [ "filter_bytecode_generator_unittest.cc", "filter_bytecode_parser_unittest.cc", + "filter_util_unittest.cc", + "message_filter_unittest.cc", "message_tokenizer_unittest.cc", ] } +if (enable_perfetto_benchmarks) { + source_set("benchmarks") { + testonly = true + deps = [ + ":message_filter", + "../../../gn:benchmark", + "../../../gn:default_deps", + "../../base", + "../../base:test_support", + ] + sources = [ "message_filter_benchmark.cc" ] + } +} + perfetto_fuzzer_test("protozero_bytecode_parser_fuzzer") { sources = [ "filter_bytecode_parser_fuzzer.cc" ] deps = [ ":bytecode_parser", + "..:protozero", "../../../gn:default_deps", + "../../base", + ] +} + +perfetto_fuzzer_test("protozero_message_filter_fuzzer") { + sources = [ "message_filter_fuzzer.cc" ] + deps = [ + ":message_filter", + "..:protozero", + "../../../gn:default_deps", + "../../base", ] } diff --git a/src/protozero/filtering/filter_bytecode_common.h b/src/protozero/filtering/filter_bytecode_common.h new file mode 100644 index 000000000..9c0e6f238 --- /dev/null +++ b/src/protozero/filtering/filter_bytecode_common.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_PROTOZERO_FILTERING_FILTER_BYTECODE_COMMON_H_ +#define SRC_PROTOZERO_FILTERING_FILTER_BYTECODE_COMMON_H_ + +#include <stdint.h> + +namespace protozero { + +enum FilterOpcode : uint32_t { + // The immediate value is 0 in this case. + kFilterOpcode_EndOfMessage = 0, + + // The immediate value is the id of the allowed field. + kFilterOpcode_SimpleField = 1, + + // The immediate value is the start of the range. The next word (without + // any shifting) is the length of the range. + kFilterOpcode_SimpleFieldRange = 2, + + // The immediate value is the id of the allowed field. The next word + // (without any shifting) is the index of the filter that should be used to + // recurse into the nested message. + kFilterOpcode_NestedField = 3, +}; +} // namespace protozero + +#endif // SRC_PROTOZERO_FILTERING_FILTER_BYTECODE_COMMON_H_ diff --git a/src/protozero/filtering/filter_bytecode_generator.cc b/src/protozero/filtering/filter_bytecode_generator.cc index df3a80e62..936286915 100644 --- a/src/protozero/filtering/filter_bytecode_generator.cc +++ b/src/protozero/filtering/filter_bytecode_generator.cc @@ -17,22 +17,20 @@ #include "src/protozero/filtering/filter_bytecode_generator.h" #include "perfetto/base/logging.h" +#include "perfetto/ext/base/hash.h" #include "perfetto/protozero/packed_repeated_fields.h" #include "perfetto/protozero/proto_utils.h" #include "perfetto/protozero/scattered_heap_buffer.h" - -#include "protos/perfetto/config/proto_filter.pbzero.h" +#include "src/protozero/filtering/filter_bytecode_common.h" namespace protozero { -using ProtoFilter = perfetto::protos::pbzero::ProtoFilter; - FilterBytecodeGenerator::FilterBytecodeGenerator() = default; FilterBytecodeGenerator::~FilterBytecodeGenerator() = default; void FilterBytecodeGenerator::EndMessage() { endmessage_called_ = true; - bytecode_.push_back(ProtoFilter::FILTER_OPCODE_END_OF_MESSAGE); + bytecode_.push_back(kFilterOpcode_EndOfMessage); last_field_id_ = 0; ++num_messages_; } @@ -40,7 +38,7 @@ void FilterBytecodeGenerator::EndMessage() { // Allows a simple field (varint, fixed32/64, string or bytes). void FilterBytecodeGenerator::AddSimpleField(uint32_t field_id) { PERFETTO_CHECK(field_id > last_field_id_); - bytecode_.push_back(field_id << 3 | ProtoFilter::FILTER_OPCODE_SIMPLE_FIELD); + bytecode_.push_back(field_id << 3 | kFilterOpcode_SimpleField); last_field_id_ = field_id; endmessage_called_ = false; } @@ -52,8 +50,7 @@ void FilterBytecodeGenerator::AddSimpleFieldRange(uint32_t range_start, uint32_t range_len) { PERFETTO_CHECK(range_start > last_field_id_); PERFETTO_CHECK(range_len > 0); - bytecode_.push_back(range_start << 3 | - ProtoFilter::FILTER_OPCODE_SIMPLE_FIELD_RANGE); + bytecode_.push_back(range_start << 3 | kFilterOpcode_SimpleFieldRange); bytecode_.push_back(range_len); last_field_id_ = range_start + range_len - 1; endmessage_called_ = false; @@ -67,26 +64,28 @@ void FilterBytecodeGenerator::AddSimpleFieldRange(uint32_t range_start, void FilterBytecodeGenerator::AddNestedField(uint32_t field_id, uint32_t message_index) { PERFETTO_CHECK(field_id > last_field_id_); - bytecode_.push_back(field_id << 3 | ProtoFilter::FILTER_OPCODE_NESTED_FIELD); + bytecode_.push_back(field_id << 3 | kFilterOpcode_NestedField); bytecode_.push_back(message_index); last_field_id_ = field_id; max_msg_index_ = std::max(max_msg_index_, message_index); endmessage_called_ = false; } -// Returns the proto-encoded bytes for a perfetto.protos.ProtoFilter message -// (see proto_filter.proto). The returned string can be passed to -// FilterBytecodeParser.Load(). +// Returns the bytes that can be used into TraceConfig.trace_filter.bytecode. +// The returned bytecode is a binary buffer which consists of a sequence of +// varints (the opcodes) and a checksum. +// The returned string can be passed as-is to FilterBytecodeParser.Load(). std::string FilterBytecodeGenerator::Serialize() { PERFETTO_CHECK(endmessage_called_); PERFETTO_CHECK(max_msg_index_ < num_messages_); protozero::PackedVarInt words; - for (uint32_t word : bytecode_) + perfetto::base::Hash hasher; + for (uint32_t word : bytecode_) { words.Append(word); - - protozero::HeapBuffered<ProtoFilter> filter; - filter->set_bytecode(words); - return filter.SerializeAsString(); + hasher.Update(word); + } + words.Append(static_cast<uint32_t>(hasher.digest())); + return std::string(reinterpret_cast<const char*>(words.data()), words.size()); } } // namespace protozero diff --git a/src/protozero/filtering/filter_bytecode_generator.h b/src/protozero/filtering/filter_bytecode_generator.h index 481d39fb4..809799c24 100644 --- a/src/protozero/filtering/filter_bytecode_generator.h +++ b/src/protozero/filtering/filter_bytecode_generator.h @@ -57,8 +57,8 @@ class FilterBytecodeGenerator { // made). void AddNestedField(uint32_t field_id, uint32_t message_index); - // Returns the proto-encoded bytes for a perfetto.protos.ProtoFilter message - // (see proto_filter.proto). The returned string can be passed to + // Returns the filter bytecode, which is a buffer containing a sequence of + // varints and a checksum. The returned string can be passed to // FilterBytecodeParser.Load(). std::string Serialize(); diff --git a/src/protozero/filtering/filter_bytecode_generator_unittest.cc b/src/protozero/filtering/filter_bytecode_generator_unittest.cc index 95181f578..650120b84 100644 --- a/src/protozero/filtering/filter_bytecode_generator_unittest.cc +++ b/src/protozero/filtering/filter_bytecode_generator_unittest.cc @@ -21,8 +21,6 @@ #include "src/protozero/filtering/filter_bytecode_generator.h" #include "src/protozero/filtering/filter_bytecode_parser.h" -#include "protos/perfetto/config/proto_filter.pbzero.h" - // This file tests the generator, assuming the parser is good. // The parser is tested separately (without the generator) in // filter_bytecode_parser_unittest.cc diff --git a/src/protozero/filtering/filter_bytecode_parser.cc b/src/protozero/filtering/filter_bytecode_parser.cc index 1a1d32fe8..39c8bb640 100644 --- a/src/protozero/filtering/filter_bytecode_parser.cc +++ b/src/protozero/filtering/filter_bytecode_parser.cc @@ -17,15 +17,18 @@ #include "src/protozero/filtering/filter_bytecode_parser.h" #include "perfetto/base/logging.h" +#include "perfetto/ext/base/hash.h" #include "perfetto/protozero/packed_repeated_fields.h" +#include "perfetto/protozero/proto_decoder.h" #include "perfetto/protozero/proto_utils.h" - -#include "protos/perfetto/config/proto_filter.pbzero.h" +#include "src/protozero/filtering/filter_bytecode_common.h" namespace protozero { void FilterBytecodeParser::Reset() { + bool suppress = suppress_logs_for_fuzzer_; *this = FilterBytecodeParser(); + suppress_logs_for_fuzzer_ = suppress; } bool FilterBytecodeParser::Load(const void* filter_data, size_t len) { @@ -37,22 +40,37 @@ bool FilterBytecodeParser::Load(const void* filter_data, size_t len) { return res; } -bool FilterBytecodeParser::LoadInternal(const uint8_t* filter_data, +bool FilterBytecodeParser::LoadInternal(const uint8_t* bytecode_data, size_t len) { - using ProtoFilter = perfetto::protos::pbzero::ProtoFilter; // First unpack the varints into a plain uint32 vector, so it's easy to // iterate through them and look ahead. - std::vector<uint32_t> bytecode; - { - ProtoFilter::Decoder dec(filter_data, len); - bool packed_parse_err = false; - bytecode.reserve(len); // An overestimation, but avoids reallocations. - for (auto it = dec.bytecode(&packed_parse_err); it; ++it) - bytecode.emplace_back(*it); - if (packed_parse_err) - return false; + std::vector<uint32_t> words; + bool packed_parse_err = false; + words.reserve(len); // An overestimation, but avoids reallocations. + using BytecodeDecoder = + PackedRepeatedFieldIterator<proto_utils::ProtoWireType::kVarInt, + uint32_t>; + for (BytecodeDecoder it(bytecode_data, len, &packed_parse_err); it; ++it) + words.emplace_back(*it); + + if (packed_parse_err || words.empty()) + return false; + + perfetto::base::Hash hasher; + for (size_t i = 0; i < words.size() - 1; ++i) + hasher.Update(words[i]); + + uint32_t expected_csum = static_cast<uint32_t>(hasher.digest()); + if (expected_csum != words.back()) { + if (!suppress_logs_for_fuzzer_) { + PERFETTO_ELOG("Filter bytecode checksum failed. Expected: %x, actual: %x", + expected_csum, words.back()); + } + return false; } + words.pop_back(); // Pop the checksum. + // Temporay storage for each message. Cleared on every END_OF_MESSAGE. std::vector<uint32_t> direct_indexed_fields; std::vector<uint32_t> ranges; @@ -73,26 +91,26 @@ bool FilterBytecodeParser::LoadInternal(const uint8_t* filter_data, ranges.emplace_back(kAllowed | msg_id); }; - for (size_t i = 0; i < bytecode.size(); ++i) { - const uint32_t word = bytecode[i]; - const bool has_next_word = i < bytecode.size() - 1; + for (size_t i = 0; i < words.size(); ++i) { + const uint32_t word = words[i]; + const bool has_next_word = i < words.size() - 1; const uint32_t opcode = word & 0x7u; const uint32_t field_id = word >> 3; - if (opcode == ProtoFilter::FILTER_OPCODE_SIMPLE_FIELD || - opcode == ProtoFilter::FILTER_OPCODE_NESTED_FIELD) { - if (field_id == 0) { - PERFETTO_DLOG("bytecode error @ word %zu, invalid field id (0)", i); - return false; - } + if (field_id == 0 && opcode != kFilterOpcode_EndOfMessage) { + PERFETTO_DLOG("bytecode error @ word %zu, invalid field id (0)", i); + return false; + } + if (opcode == kFilterOpcode_SimpleField || + opcode == kFilterOpcode_NestedField) { // Field words are organized as follow: // MSB: 1 if allowed, 0 if not allowed. // Remaining bits: // Message index in the case of nested (non-simple) messages. // 0x7f..f in the case of simple messages. uint32_t msg_id; - if (opcode == ProtoFilter::FILTER_OPCODE_SIMPLE_FIELD) { + if (opcode == kFilterOpcode_SimpleField) { msg_id = kSimpleField; } else { // FILTER_OPCODE_NESTED_FIELD // The next word in the bytecode contains the message index. @@ -101,7 +119,7 @@ bool FilterBytecodeParser::LoadInternal(const uint8_t* filter_data, i); return false; } - msg_id = bytecode[++i]; + msg_id = words[++i]; max_msg_index = std::max(max_msg_index, msg_id); } @@ -113,12 +131,12 @@ bool FilterBytecodeParser::LoadInternal(const uint8_t* filter_data, // complexity to deal with rare cases like this. add_range(field_id, field_id + 1, msg_id); } - } else if (opcode == ProtoFilter::FILTER_OPCODE_SIMPLE_FIELD_RANGE) { + } else if (opcode == kFilterOpcode_SimpleFieldRange) { if (!has_next_word) { PERFETTO_DLOG("bytecode error @ word %zu: unterminated range", i); return false; } - const uint32_t range_len = bytecode[++i]; + const uint32_t range_len = words[++i]; const uint32_t range_end = field_id + range_len; // STL-style, excl. uint32_t id = field_id; @@ -132,7 +150,7 @@ bool FilterBytecodeParser::LoadInternal(const uint8_t* filter_data, PERFETTO_DCHECK(id >= kDirectlyIndexLimit || id == range_end); if (id < range_end) add_range(id, range_end, kSimpleField); - } else if (opcode == ProtoFilter::FILTER_OPCODE_END_OF_MESSAGE) { + } else if (opcode == kFilterOpcode_EndOfMessage) { // For each message append: // 1. The "header" word telling how many directly indexed fields there // are. @@ -182,7 +200,8 @@ FilterBytecodeParser::QueryResult FilterBytecodeParser::Query( // bytecode. PERFETTO_DCHECK(start_offset < words_.size()); const uint32_t* word = &words_[start_offset]; - const uint32_t* const end = &words_[message_offset_[msg_index + 1]]; + const uint32_t end_off = message_offset_[msg_index + 1]; + const uint32_t* const end = words_.data() + end_off; PERFETTO_DCHECK(end > word && end <= words_.data() + words_.size()); const uint32_t num_directly_indexed = *(word++); PERFETTO_DCHECK(num_directly_indexed <= kDirectlyIndexLimit); diff --git a/src/protozero/filtering/filter_bytecode_parser.h b/src/protozero/filtering/filter_bytecode_parser.h index c71ec301f..b2378caca 100644 --- a/src/protozero/filtering/filter_bytecode_parser.h +++ b/src/protozero/filtering/filter_bytecode_parser.h @@ -24,10 +24,9 @@ namespace protozero { -// Loads the proto-encoded bytecode (see proto_filter.proto) in memory and -// allows fast lookups for tuples (msg_index, field_id) to tell if a given field -// should be allowed or not and, in the case of nested fields, what is the next -// message index to recurse into. +// Loads the proto-encoded bytecode in memory and allows fast lookups for tuples +// (msg_index, field_id) to tell if a given field should be allowed or not and, +// in the case of nested fields, what is the next message index to recurse into. // This class does two things: // 1. Expands the array of varint from the proto into a vector<uint32_t>. This // is to avoid performing varint decoding on every lookup, at the cost of @@ -58,8 +57,8 @@ class FilterBytecodeParser { bool simple_field() const { return nested_msg_index == kSimpleField; } }; - // Loads a filter. The passed data must be proto-encoded bytes for a - // perfetto.protos.ProtoFilter message. + // Loads a filter. The filter data consists of a sequence of varints which + // contains the filter opcodes and a final checksum. bool Load(const void* filter_data, size_t len); // Checks wheter a given field is allowed or not. @@ -68,6 +67,7 @@ class FilterBytecodeParser { QueryResult Query(uint32_t msg_index, uint32_t field_id); void Reset(); + void set_suppress_logs_for_fuzzer(bool x) { suppress_logs_for_fuzzer_ = x; } private: static constexpr uint32_t kDirectlyIndexLimit = 128; @@ -115,6 +115,8 @@ class FilterBytecodeParser { // Nth message start. // message_offset_.size() - 2 == the max message id that can be parsed. std::vector<uint32_t> message_offset_; + + bool suppress_logs_for_fuzzer_ = false; }; } // namespace protozero diff --git a/src/protozero/filtering/filter_bytecode_parser_fuzzer.cc b/src/protozero/filtering/filter_bytecode_parser_fuzzer.cc index 33dc5c911..578f6c919 100644 --- a/src/protozero/filtering/filter_bytecode_parser_fuzzer.cc +++ b/src/protozero/filtering/filter_bytecode_parser_fuzzer.cc @@ -18,14 +18,46 @@ #include <stdint.h> #include <string.h> +#include "perfetto/ext/base/hash.h" + +#include "perfetto/protozero/packed_repeated_fields.h" #include "src/protozero/filtering/filter_bytecode_parser.h" namespace protozero { namespace { +// This function gives a little help to the fuzzer. The bytecode is really a +// sequence of varint-encoded uint32 words, with a FNV1a checksum at the end. +// It's very unlikely that the fuzzer on its own can work out the checksum, so +// most fuzzer inputs are doomed to fail the checksum verification. +// This takes the fuzzer input and builds a more plausible bytecode. +void LoadBytecodeWithChecksum(FilterBytecodeParser* parser, + const uint8_t* data, + size_t size) { + protozero::PackedVarInt words; + perfetto::base::Hash hasher; + for (size_t i = 0; i < size; i += sizeof(uint32_t)) { + uint32_t word = 0; + memcpy(&word, data, sizeof(uint32_t)); + words.Append(word); + hasher.Update(word); + } + words.Append(static_cast<uint32_t>(hasher.digest())); + parser->Load(words.data(), words.size()); +} + int FuzzBytecodeParser(const uint8_t* data, size_t size) { FilterBytecodeParser parser; - parser.Load(data, size > 8 ? size - 8 : size); + parser.set_suppress_logs_for_fuzzer(true); + + if (size > 4 && data[0] < 192) { + // 75% of the times use the LoadBytecodeWithChecksum() which helps the + // fuzzer passing the checksum verification. + LoadBytecodeWithChecksum(&parser, data + 1, size - 1); + } else { + // In the remaining 25%, pass completely arbitrary inputs. + parser.Load(data, size); + } // Smoke testing with known problematic values for (uint32_t msg_index = 0; msg_index < 3; ++msg_index) { diff --git a/src/protozero/filtering/filter_bytecode_parser_unittest.cc b/src/protozero/filtering/filter_bytecode_parser_unittest.cc index 28e4e4b28..5b67ded64 100644 --- a/src/protozero/filtering/filter_bytecode_parser_unittest.cc +++ b/src/protozero/filtering/filter_bytecode_parser_unittest.cc @@ -16,33 +16,32 @@ #include "test/gtest_and_gmock.h" +#include "perfetto/ext/base/hash.h" #include "perfetto/protozero/packed_repeated_fields.h" #include "perfetto/protozero/scattered_heap_buffer.h" +#include "src/protozero/filtering/filter_bytecode_common.h" #include "src/protozero/filtering/filter_bytecode_parser.h" -#include "protos/perfetto/config/proto_filter.pbzero.h" - namespace protozero { namespace { -using PF = perfetto::protos::pbzero::ProtoFilter; bool LoadBytecode(FilterBytecodeParser* parser, std::initializer_list<uint32_t> bytecode) { + perfetto::base::Hash hasher; protozero::PackedVarInt words; - for (uint32_t w : bytecode) + for (uint32_t w : bytecode) { words.Append(w); - - HeapBuffered<PF> filter; - filter->set_bytecode(words); - auto filter_msg = filter.SerializeAsArray(); - return parser->Load(filter_msg.data(), filter_msg.size()); + hasher.Update(w); + } + words.Append(static_cast<uint32_t>(hasher.digest())); + return parser->Load(words.data(), words.size()); } TEST(FilterBytecodeParserTest, ParserSimpleFields) { FilterBytecodeParser parser; - EXPECT_TRUE(parser.Load(nullptr, 0)); + EXPECT_FALSE(parser.Load(nullptr, 0)); EXPECT_FALSE(parser.Query(42, 42).allowed); EXPECT_TRUE(LoadBytecode(&parser, {})); @@ -54,12 +53,12 @@ TEST(FilterBytecodeParserTest, ParserSimpleFields) { EXPECT_FALSE(parser.Query(42, 42).allowed); // An invalid field_id (0) in bytecode should cause a parse failure. - EXPECT_FALSE(LoadBytecode(&parser, {PF::FILTER_OPCODE_SIMPLE_FIELD | 0, - PF::FILTER_OPCODE_END_OF_MESSAGE})); + EXPECT_FALSE(LoadBytecode( + &parser, {kFilterOpcode_SimpleField | 0, kFilterOpcode_EndOfMessage})); // A valid bytecode that has only one field. - EXPECT_TRUE(LoadBytecode(&parser, {PF::FILTER_OPCODE_SIMPLE_FIELD | (2u << 3), - PF::FILTER_OPCODE_END_OF_MESSAGE})); + EXPECT_TRUE(LoadBytecode(&parser, {kFilterOpcode_SimpleField | (2u << 3), + kFilterOpcode_EndOfMessage})); EXPECT_FALSE(parser.Query(0, 0).allowed); EXPECT_FALSE(parser.Query(0, 1).allowed); EXPECT_TRUE(parser.Query(0, 2).allowed); @@ -71,12 +70,11 @@ TEST(FilterBytecodeParserTest, ParserSimpleFields) { EXPECT_FALSE(parser.Query(1, 3).allowed); // A valid bytecode that has few sparse fields < 128. - EXPECT_TRUE( - LoadBytecode(&parser, {PF::FILTER_OPCODE_SIMPLE_FIELD | (1u << 3), - PF::FILTER_OPCODE_SIMPLE_FIELD | (7u << 3), - PF::FILTER_OPCODE_SIMPLE_FIELD | (8u << 3), - PF::FILTER_OPCODE_SIMPLE_FIELD | (127u << 3), - PF::FILTER_OPCODE_END_OF_MESSAGE})); + EXPECT_TRUE(LoadBytecode(&parser, {kFilterOpcode_SimpleField | (1u << 3), + kFilterOpcode_SimpleField | (7u << 3), + kFilterOpcode_SimpleField | (8u << 3), + kFilterOpcode_SimpleField | (127u << 3), + kFilterOpcode_EndOfMessage})); EXPECT_FALSE(parser.Query(0, 0).allowed); EXPECT_TRUE(parser.Query(0, 1).allowed); EXPECT_FALSE(parser.Query(0, 2).allowed); @@ -90,11 +88,10 @@ TEST(FilterBytecodeParserTest, ParserSimpleFields) { EXPECT_FALSE(parser.Query(0, 128).allowed); // A valid bytecode that has only fields > 128. - EXPECT_TRUE( - LoadBytecode(&parser, {PF::FILTER_OPCODE_SIMPLE_FIELD | (1000u << 3), - PF::FILTER_OPCODE_SIMPLE_FIELD | (1001u << 3), - PF::FILTER_OPCODE_SIMPLE_FIELD | (2000u << 3), - PF::FILTER_OPCODE_END_OF_MESSAGE})); + EXPECT_TRUE(LoadBytecode(&parser, {kFilterOpcode_SimpleField | (1000u << 3), + kFilterOpcode_SimpleField | (1001u << 3), + kFilterOpcode_SimpleField | (2000u << 3), + kFilterOpcode_EndOfMessage})); for (uint32_t i = 0; i < 1000; ++i) EXPECT_FALSE(parser.Query(0, i).allowed); EXPECT_TRUE(parser.Query(0, 1000).allowed); @@ -109,28 +106,28 @@ TEST(FilterBytecodeParserTest, ParserSimpleRanges) { FilterBytecodeParser parser; // Invalid, range length missing. - EXPECT_FALSE(LoadBytecode( - &parser, { - PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (2u << 3), - })); + EXPECT_FALSE( + LoadBytecode(&parser, { + kFilterOpcode_SimpleFieldRange | (2u << 3), + })); // Borderline valid: range length = 0. - EXPECT_TRUE(LoadBytecode( - &parser, {PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (2u << 3), 0u, - PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (127u << 3), 0u, - PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (128u << 3), 0u, - PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (128u << 3), 0u, - PF::FILTER_OPCODE_END_OF_MESSAGE})); + EXPECT_TRUE( + LoadBytecode(&parser, {kFilterOpcode_SimpleFieldRange | (2u << 3), 0u, + kFilterOpcode_SimpleFieldRange | (127u << 3), 0u, + kFilterOpcode_SimpleFieldRange | (128u << 3), 0u, + kFilterOpcode_SimpleFieldRange | (128u << 3), 0u, + kFilterOpcode_EndOfMessage})); for (uint32_t i = 0; i < 130; ++i) EXPECT_FALSE(parser.Query(0, i).allowed) << i; // A valid bytecode with two ranges [2,2], [10, 14]. EXPECT_TRUE( - LoadBytecode(&parser, {PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (2u << 3), + LoadBytecode(&parser, {kFilterOpcode_SimpleFieldRange | (2u << 3), 1u, // length of the range, - PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (10u << 3), + kFilterOpcode_SimpleFieldRange | (10u << 3), 5u, // length of the range, - PF::FILTER_OPCODE_END_OF_MESSAGE})); + kFilterOpcode_EndOfMessage})); EXPECT_FALSE(parser.Query(0, 0).allowed); EXPECT_FALSE(parser.Query(0, 1).allowed); EXPECT_TRUE(parser.Query(0, 2).allowed); @@ -147,17 +144,17 @@ TEST(FilterBytecodeParserTest, ParserSimpleFieldsAndRanges) { // Borderline valid: range length = 0. EXPECT_TRUE( - LoadBytecode(&parser, {PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (1u << 3), + LoadBytecode(&parser, {kFilterOpcode_SimpleFieldRange | (1u << 3), 2u, // [1,2] - PF::FILTER_OPCODE_SIMPLE_FIELD | (4u << 3), + kFilterOpcode_SimpleField | (4u << 3), - PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (126u << 3), + kFilterOpcode_SimpleFieldRange | (126u << 3), 4u, // [126, 129] - PF::FILTER_OPCODE_SIMPLE_FIELD | (150u << 3), + kFilterOpcode_SimpleField | (150u << 3), - PF::FILTER_OPCODE_END_OF_MESSAGE})); + kFilterOpcode_EndOfMessage})); EXPECT_TRUE(parser.Query(0, 1).allowed); EXPECT_TRUE(parser.Query(0, 2).allowed); EXPECT_FALSE(parser.Query(0, 3).allowed); @@ -175,42 +172,41 @@ TEST(FilterBytecodeParserTest, ParserNestedMessages) { // Invalid because there are 1 messages in total, and message index 1 is // out of range. - EXPECT_FALSE( - LoadBytecode(&parser, {PF::FILTER_OPCODE_NESTED_FIELD | (4u << 3), - 1u, // message index - PF::FILTER_OPCODE_END_OF_MESSAGE})); + EXPECT_FALSE(LoadBytecode(&parser, {kFilterOpcode_NestedField | (4u << 3), + 1u, // message index + kFilterOpcode_EndOfMessage})); // A valid bytecode consisting of 4 message, with recursive / cylical // dependencies between them. EXPECT_TRUE(LoadBytecode( &parser, { // Message 0 (root). - PF::FILTER_OPCODE_SIMPLE_FIELD_RANGE | (1u << 3), + kFilterOpcode_SimpleFieldRange | (1u << 3), 2u, // [1,2] - PF::FILTER_OPCODE_NESTED_FIELD | (4u << 3), + kFilterOpcode_NestedField | (4u << 3), 3u, // message index - PF::FILTER_OPCODE_SIMPLE_FIELD | (10u << 3), - PF::FILTER_OPCODE_NESTED_FIELD | (127u << 3), + kFilterOpcode_SimpleField | (10u << 3), + kFilterOpcode_NestedField | (127u << 3), 1u, // message index - PF::FILTER_OPCODE_NESTED_FIELD | (128u << 3), + kFilterOpcode_NestedField | (128u << 3), 2u, // message index - PF::FILTER_OPCODE_END_OF_MESSAGE, + kFilterOpcode_EndOfMessage, // Message 1. - PF::FILTER_OPCODE_NESTED_FIELD | (2u << 3), + kFilterOpcode_NestedField | (2u << 3), 1u, // message index (recurse onto itself), - PF::FILTER_OPCODE_SIMPLE_FIELD | (11u << 3), - PF::FILTER_OPCODE_END_OF_MESSAGE, + kFilterOpcode_SimpleField | (11u << 3), + kFilterOpcode_EndOfMessage, // Message 2. - PF::FILTER_OPCODE_NESTED_FIELD | (2u << 3), + kFilterOpcode_NestedField | (2u << 3), 3u, // message index. - PF::FILTER_OPCODE_END_OF_MESSAGE, + kFilterOpcode_EndOfMessage, // Message 3. - PF::FILTER_OPCODE_NESTED_FIELD | (2u << 3), + kFilterOpcode_NestedField | (2u << 3), 2u, // message index (create a cycle, 2->3, 3->2). - PF::FILTER_OPCODE_END_OF_MESSAGE, + kFilterOpcode_EndOfMessage, })); // Query message 0 fields. diff --git a/src/protozero/filtering/filter_util.cc b/src/protozero/filtering/filter_util.cc new file mode 100644 index 000000000..94d436e7a --- /dev/null +++ b/src/protozero/filtering/filter_util.cc @@ -0,0 +1,323 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/protozero/filtering/filter_util.h" + +#include <algorithm> +#include <map> +#include <memory> +#include <set> + +#include <google/protobuf/compiler/importer.h> + +#include "perfetto/base/build_config.h" +#include "perfetto/ext/base/file_utils.h" +#include "perfetto/ext/base/getopt.h" +#include "perfetto/ext/base/string_utils.h" +#include "perfetto/ext/base/version.h" +#include "perfetto/protozero/proto_utils.h" +#include "src/protozero/filtering/filter_bytecode_generator.h" + +namespace protozero { + +namespace { + +class MultiFileErrorCollectorImpl + : public google::protobuf::compiler::MultiFileErrorCollector { + public: + ~MultiFileErrorCollectorImpl() override; + void AddError(const std::string&, int, int, const std::string&) override; + void AddWarning(const std::string&, int, int, const std::string&) override; +}; + +MultiFileErrorCollectorImpl::~MultiFileErrorCollectorImpl() = default; + +void MultiFileErrorCollectorImpl::AddError(const std::string& filename, + int line, + int column, + const std::string& message) { + PERFETTO_ELOG("Error %s %d:%d: %s", filename.c_str(), line, column, + message.c_str()); +} + +void MultiFileErrorCollectorImpl::AddWarning(const std::string& filename, + int line, + int column, + const std::string& message) { + PERFETTO_ELOG("Warning %s %d:%d: %s", filename.c_str(), line, column, + message.c_str()); +} + +} // namespace + +FilterUtil::FilterUtil() = default; +FilterUtil::~FilterUtil() = default; + +bool FilterUtil::LoadMessageDefinition(const std::string& proto_file, + const std::string& root_message, + const std::string& proto_dir_path) { + // The protobuf compiler doesn't like backslashes and prints an error like: + // Error C:\it7mjanpw3\perfetto-a16500 -1:0: Backslashes, consecutive slashes, + // ".", or ".." are not allowed in the virtual path. + // Given that C:\foo\bar is a legit path on windows, fix it at this level + // because the problem is really the protobuf compiler being too picky. + static auto normalize_for_win = [](const std::string& path) { +#if PERFETTO_BUILDFLAG(PERFETTO_OS_WIN) + return perfetto::base::ReplaceAll(path, "\\", "/"); +#else + return path; +#endif + }; + + google::protobuf::compiler::DiskSourceTree dst; +#if PERFETTO_BUILDFLAG(PERFETTO_OS_WIN) + // If the path is absolute, maps "C:/" -> "C:/" (without hardcoding 'C'). + if (proto_file.size() > 3 && proto_file[1] == ':') { + char win_drive[4]; + sprintf(win_drive, "%c:/", proto_file[0]); + dst.MapPath(win_drive, win_drive); + } +#endif + dst.MapPath("/", "/"); // We might still need this on Win under cygwin. + dst.MapPath("", normalize_for_win(proto_dir_path)); + MultiFileErrorCollectorImpl mfe; + google::protobuf::compiler::Importer importer(&dst, &mfe); + const google::protobuf::FileDescriptor* root_file = + importer.Import(normalize_for_win(proto_file)); + const google::protobuf::Descriptor* root_msg = nullptr; + if (!root_message.empty()) { + root_msg = importer.pool()->FindMessageTypeByName(root_message); + } else if (root_file->message_type_count() > 0) { + // The user didn't specfy the root type. Pick the first type in the file, + // most times it's the right guess. + root_msg = root_file->message_type(0); + if (root_msg) + PERFETTO_LOG( + "The guessed root message name is \"%s\". Pass -r com.MyName to " + "override", + root_msg->full_name().c_str()); + } + + if (!root_msg) { + PERFETTO_ELOG("Could not find the root message \"%s\" in %s", + root_message.c_str(), proto_file.c_str()); + return false; + } + + // |descriptors_by_full_name| is passed by argument rather than being a member + // field so that we don't risk leaving it out of sync (and depending on it in + // future without realizing) when performing the Dedupe() pass. + DescriptorsByNameMap descriptors_by_full_name; + ParseProtoDescriptor(root_msg, &descriptors_by_full_name); + return true; +} + +// Generates a Message object for the given libprotobuf message descriptor. +// Recurses as needed into nested fields. +FilterUtil::Message* FilterUtil::ParseProtoDescriptor( + const google::protobuf::Descriptor* proto, + DescriptorsByNameMap* descriptors_by_full_name) { + auto descr_it = descriptors_by_full_name->find(proto->full_name()); + if (descr_it != descriptors_by_full_name->end()) + return descr_it->second; + + descriptors_.emplace_back(); + Message* msg = &descriptors_.back(); + msg->full_name = proto->full_name(); + (*descriptors_by_full_name)[msg->full_name] = msg; + for (int i = 0; i < proto->field_count(); ++i) { + const auto* proto_field = proto->field(i); + const uint32_t field_id = static_cast<uint32_t>(proto_field->number()); + PERFETTO_CHECK(msg->fields.count(field_id) == 0); + auto& field = msg->fields[field_id]; + field.name = proto_field->name(); + field.type = proto_field->type_name(); + if (proto_field->message_type()) { + msg->has_nested_fields = true; + // Recurse. + field.nested_type = ParseProtoDescriptor(proto_field->message_type(), + descriptors_by_full_name); + } + } + return msg; +} + +void FilterUtil::Dedupe() { + std::map<std::string /*identity*/, Message*> index; + + std::map<Message*, Message*> dupe_graph; // K,V: K shall be duped against V. + + // As a first pass, generate an |identity| string for each leaf message. The + // identity is simply the comma-separated stringification of its field ids. + // If another message with the same identity exists, add an edge to the graph. + const size_t initial_count = descriptors_.size(); + size_t field_count = 0; + for (auto& descr : descriptors_) { + if (descr.has_nested_fields) + continue; // Dedupe only leaf messages without nested fields. + std::string identity; + for (const auto& id_and_field : descr.fields) + identity.append(std::to_string(id_and_field.first) + ","); + auto it_and_inserted = index.emplace(identity, &descr); + if (!it_and_inserted.second) { + // insertion failed, a dupe exists already. + Message* dupe_against = it_and_inserted.first->second; + dupe_graph.emplace(&descr, dupe_against); + } + } + + // Now apply de-duplications by re-directing the nested_type pointer to the + // equivalent descriptors that have the same set of allowed field ids. + std::set<Message*> referenced_descriptors; + referenced_descriptors.emplace(&descriptors_.front()); // The root. + for (auto& descr : descriptors_) { + for (auto& id_and_field : descr.fields) { + Message* target = id_and_field.second.nested_type; + if (!target) + continue; // Only try to dedupe nested types. + auto it = dupe_graph.find(target); + if (it == dupe_graph.end()) { + referenced_descriptors.emplace(target); + continue; + } + ++field_count; + // Replace with the dupe. + id_and_field.second.nested_type = it->second; + } // for (nested_fields). + } // for (descriptors_). + + // Remove unreferenced descriptors. We should much rather crash in the case of + // a logic bug rathern than trying to use them but don't emit them. + size_t removed_count = 0; + for (auto it = descriptors_.begin(); it != descriptors_.end();) { + if (referenced_descriptors.count(&*it)) { + ++it; + } else { + ++removed_count; + it = descriptors_.erase(it); + } + } + PERFETTO_LOG( + "Deduplication removed %zu duped descriptors out of %zu descriptors from " + "%zu fields", + removed_count, initial_count, field_count); +} + +// Prints the list of messages and fields in a diff-friendly text format. +void FilterUtil::PrintAsText() { + using perfetto::base::StripPrefix; + const std::string& root_name = descriptors_.front().full_name; + std::string root_prefix = root_name.substr(0, root_name.rfind('.')); + if (!root_prefix.empty()) + root_prefix.append("."); + + for (const auto& descr : descriptors_) { + for (const auto& id_and_field : descr.fields) { + const uint32_t field_id = id_and_field.first; + const auto& field = id_and_field.second; + const Message* nested_type = id_and_field.second.nested_type; + auto stripped_name = StripPrefix(descr.full_name, root_prefix); + std::string stripped_nested = + nested_type ? StripPrefix(nested_type->full_name, root_prefix) : ""; + printf("%-60s %3u %-8s %-32s %s\n", stripped_name.c_str(), field_id, + field.type.c_str(), field.name.c_str(), stripped_nested.c_str()); + } + } +} + +std::string FilterUtil::GenerateFilterBytecode() { + protozero::FilterBytecodeGenerator bytecode_gen; + + // Assign indexes to descriptors, simply by counting them in order; + std::map<Message*, uint32_t> descr_to_idx; + for (auto& descr : descriptors_) + descr_to_idx[&descr] = static_cast<uint32_t>(descr_to_idx.size()); + + for (auto& descr : descriptors_) { + for (auto it = descr.fields.begin(); it != descr.fields.end();) { + uint32_t field_id = it->first; + const Message::Field& field = it->second; + if (field.nested_type) { + // Append the index of the target submessage. + PERFETTO_CHECK(descr_to_idx.count(field.nested_type)); + uint32_t nested_msg_index = descr_to_idx[field.nested_type]; + bytecode_gen.AddNestedField(field_id, nested_msg_index); + ++it; + continue; + } + // Simple field. Lookahead to see if we have a range of contiguous simple + // fields. + for (uint32_t range_len = 1;; ++range_len) { + ++it; + if (it != descr.fields.end() && it->first == field_id + range_len && + it->second.is_simple()) { + continue; + } + // At this point it points to either the end() of the vector or a + // non-contiguous or non-simple field (which will be picked up by the + // next iteration). + if (range_len == 1) { + bytecode_gen.AddSimpleField(field_id); + } else { + bytecode_gen.AddSimpleFieldRange(field_id, range_len); + } + break; + } // for (range_len) + } // for (descr.fields) + bytecode_gen.EndMessage(); + } // for (descriptors) + return bytecode_gen.Serialize(); +} + +std::string FilterUtil::LookupField(const std::string& varint_encoded_path) { + const uint8_t* ptr = + reinterpret_cast<const uint8_t*>(varint_encoded_path.data()); + const uint8_t* const end = ptr + varint_encoded_path.size(); + + std::vector<uint32_t> fields; + while (ptr < end) { + uint64_t varint; + const uint8_t* next = proto_utils::ParseVarInt(ptr, end, &varint); + PERFETTO_CHECK(next != ptr); + fields.emplace_back(static_cast<uint32_t>(varint)); + ptr = next; + } + return LookupField(fields.data(), fields.size()); +} + +std::string FilterUtil::LookupField(const uint32_t* field_ids, + size_t num_fields) { + const Message* msg = descriptors_.empty() ? nullptr : &descriptors_.front(); + std::string res; + for (size_t i = 0; i < num_fields; ++i) { + const uint32_t field_id = field_ids[i]; + const Message::Field* field = nullptr; + if (msg) { + auto it = msg->fields.find(field_id); + field = it == msg->fields.end() ? nullptr : &it->second; + } + res.append("."); + if (field) { + res.append(field->name); + msg = field->nested_type; + } else { + res.append(std::to_string(field_id)); + } + } + return res; +} + +} // namespace protozero diff --git a/src/protozero/filtering/filter_util.h b/src/protozero/filtering/filter_util.h new file mode 100644 index 000000000..3cc7655c8 --- /dev/null +++ b/src/protozero/filtering/filter_util.h @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_PROTOZERO_FILTERING_FILTER_UTIL_H_ +#define SRC_PROTOZERO_FILTERING_FILTER_UTIL_H_ + +#include <stdint.h> + +#include <list> +#include <map> +#include <string> + +namespace google { +namespace protobuf { +class Descriptor; +} +} // namespace google + +namespace protozero { + +// Parses a .proto message definition, recursing into its sub-messages, and +// builds up a set of Messages and Field definitions. +// Depends on libprotobuf-full and should be used only in host tools. +// See the //tools/proto_filter for an executable that wraps this class with +// a cmdline interface. +class FilterUtil { + public: + FilterUtil(); + ~FilterUtil(); + + // Loads a message schema from a .proto file, recursing into nested types. + // Args: + // proto_file: path to the .proto file. + // root_message: fully qualified message name (e.g., perfetto.protos.Trace). + // If empty, the first message in the file will be used. + // proto_dir_path: the root for .proto includes. If empty uses CWD. + bool LoadMessageDefinition(const std::string& proto_file, + const std::string& root_message, + const std::string& proto_dir_path); + + // Deduplicates leaf messages having the same sets of field ids. + // It changes the internal state and affects the behavior of next calls to + // GenerateFilterBytecode() and PrintAsText(). + void Dedupe(); + + // Generates the filter bytecode for the root message previously loaded by + // LoadMessageDefinition() using FilterBytecodeGenerator. + // The returned string is a binary-encoded proto message of type + // perfetto.protos.ProtoFilter (see proto_filter.proto). + std::string GenerateFilterBytecode(); + + // Prints the list of messages and fields onto stdout in a diff-friendly text + // format. Example: + // PowerRails 2 message energy_data PowerRails.EnergyData + // PowerRails.RailDescriptor 1 uint32 index + void PrintAsText(); + + // Resolves an array of field ids into a dot-concatenated field names. + // E.g., [2,5,1] -> ".trace.packet.timestamp". + std::string LookupField(const uint32_t* field_ids, size_t num_fields); + + // Like the above but the array of field is passed as a buffer containing + // varints, e.g. "\x02\x05\0x01". + std::string LookupField(const std::string& varint_encoded_path); + + private: + struct Message { + struct Field { + std::string name; + std::string type; // "uint32", "string", "message" + // Only when type == "message". Note that when using Dedupe() this can + // be aliased against a different submessage which happens to have the + // same set of field ids. + Message* nested_type = nullptr; + bool is_simple() const { return nested_type == nullptr; } + }; + std::string full_name; // e.g., "perfetto.protos.Foo.Bar"; + std::map<uint32_t /*field_id*/, Field> fields; + + // True if at least one field has a non-null |nestd_type|. + bool has_nested_fields = false; + }; + + using DescriptorsByNameMap = std::map<std::string, Message*>; + Message* ParseProtoDescriptor(const google::protobuf::Descriptor*, + DescriptorsByNameMap*); + + // list<> because pointers need to be stable. + std::list<Message> descriptors_; +}; + +} // namespace protozero + +#endif // SRC_PROTOZERO_FILTERING_FILTER_UTIL_H_ diff --git a/src/protozero/filtering/filter_util_unittest.cc b/src/protozero/filtering/filter_util_unittest.cc new file mode 100644 index 000000000..9c7cf0d6c --- /dev/null +++ b/src/protozero/filtering/filter_util_unittest.cc @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "test/gtest_and_gmock.h" + +#include "perfetto/ext/base/file_utils.h" +#include "perfetto/ext/base/temp_file.h" +#include "src/protozero/filtering/filter_bytecode_parser.h" +#include "src/protozero/filtering/filter_util.h" + +namespace protozero { + +namespace { + +perfetto::base::TempFile MkTemp(const char* str) { + auto tmp = perfetto::base::TempFile::Create(); + perfetto::base::WriteAll(*tmp, str, strlen(str)); + perfetto::base::FlushFile(*tmp); + return tmp; +} + +TEST(SchemaParserTest, SchemaToBytecode_Simple) { + auto schema = MkTemp(R"( + syntax = "proto2"; + message Root { + optional int32 i32 = 13; + optional fixed64 f64 = 5; + optional string str = 71; + } + )"); + FilterUtil filter; + ASSERT_TRUE(filter.LoadMessageDefinition(schema.path(), "Root", "")); + std::string bytecode = filter.GenerateFilterBytecode(); + FilterBytecodeParser fbp; + ASSERT_TRUE(fbp.Load(bytecode.data(), bytecode.size())); + EXPECT_TRUE(fbp.Query(0, 13).allowed); + EXPECT_TRUE(fbp.Query(0, 13).simple_field()); + EXPECT_TRUE(fbp.Query(0, 5).allowed); + EXPECT_TRUE(fbp.Query(0, 5).simple_field()); + EXPECT_TRUE(fbp.Query(0, 71).allowed); + EXPECT_TRUE(fbp.Query(0, 71).simple_field()); + EXPECT_FALSE(fbp.Query(0, 1).allowed); + EXPECT_FALSE(fbp.Query(0, 12).allowed); + EXPECT_FALSE(fbp.Query(0, 70).allowed); +} + +TEST(SchemaParserTest, SchemaToBytecode_Nested) { + auto schema = MkTemp(R"( + syntax = "proto2"; + message Root { + message Child { + repeated fixed64 f64 = 3; + optional Child recurse = 4; + } + oneof xxx { int32 i32 = 1; } + optional Child chld = 2; + } + )"); + FilterUtil filter; + ASSERT_TRUE(filter.LoadMessageDefinition(schema.path(), "", "")); + std::string bytecode = filter.GenerateFilterBytecode(); + FilterBytecodeParser fbp; + ASSERT_TRUE(fbp.Load(bytecode.data(), bytecode.size())); + EXPECT_TRUE(fbp.Query(0, 1).allowed); + EXPECT_TRUE(fbp.Query(0, 1).simple_field()); + EXPECT_TRUE(fbp.Query(0, 2).allowed); + EXPECT_FALSE(fbp.Query(0, 2).simple_field()); + // False as those fields exist only in Child, not in the root (0). + EXPECT_FALSE(fbp.Query(0, 3).allowed); + EXPECT_FALSE(fbp.Query(0, 4).allowed); + + EXPECT_TRUE(fbp.Query(1, 3).allowed); + EXPECT_TRUE(fbp.Query(1, 3).simple_field()); + EXPECT_TRUE(fbp.Query(1, 4).allowed); + EXPECT_FALSE(fbp.Query(1, 4).simple_field()); + EXPECT_EQ(fbp.Query(1, 4).nested_msg_index, 1u); // Self +} + +TEST(SchemaParserTest, SchemaToBytecode_Dedupe) { + auto schema = MkTemp(R"( + syntax = "proto2"; + message Root { + message Nested { + message Child1 { + optional int32 f1 = 3; + optional int64 f2 = 4; + } + message Child2 { + optional string f1 = 3; + optional bytes f2 = 4; + } + message ChildNonDedupe { + optional string f1 = 3; + optional bytes f2 = 4; + optional int32 extra = 1; + } + optional Child1 chld1 = 1; + optional Child2 chld2 = 2; + optional ChildNonDedupe chld3 = 3; + } + repeated Nested nested = 1; + } + )"); + FilterUtil filter; + ASSERT_TRUE(filter.LoadMessageDefinition(schema.path(), "Root", "")); + filter.Dedupe(); + std::string bytecode = filter.GenerateFilterBytecode(); + FilterBytecodeParser fbp; + ASSERT_TRUE(fbp.Load(bytecode.data(), bytecode.size())); + + // 0: Root + EXPECT_TRUE(fbp.Query(0, 1).allowed); + EXPECT_FALSE(fbp.Query(0, 1).simple_field()); + + // 1: Nested + EXPECT_TRUE(fbp.Query(1, 1).allowed); + EXPECT_FALSE(fbp.Query(1, 1).simple_field()); + EXPECT_TRUE(fbp.Query(1, 2).allowed); + EXPECT_FALSE(fbp.Query(1, 2).simple_field()); + EXPECT_TRUE(fbp.Query(1, 3).allowed); + EXPECT_FALSE(fbp.Query(1, 3).simple_field()); + + // Check deduping. + // Fields chld1 and chld2 should point to the same sub-filter because they + // have the same field ids. + EXPECT_EQ(fbp.Query(1, 1).nested_msg_index, fbp.Query(1, 2).nested_msg_index); + + // Field chld3 should point to a different one because it has an extra field. + EXPECT_NE(fbp.Query(1, 1).nested_msg_index, fbp.Query(1, 3).nested_msg_index); +} + +TEST(SchemaParserTest, FieldLookup) { + auto schema = MkTemp(R"( + syntax = "proto2"; + message Root { + message Nested { + message Child1 { + optional int32 f1 = 3; + optional int64 f2 = 4; + repeated Child2 c2 = 5; + } + message Child2 { + optional string f3 = 6; + optional bytes f4 = 7; + repeated Child1 c1 = 8; + } + optional Child1 x1 = 1; + optional Child2 x2 = 2; + } + repeated Nested n = 1; + } + )"); + + FilterUtil filter; + ASSERT_TRUE(filter.LoadMessageDefinition(schema.path(), "Root", "")); + std::vector<uint32_t> fld; + + fld = {1, 1, 3}; + ASSERT_EQ(filter.LookupField(fld.data(), fld.size()), ".n.x1.f1"); + + fld = {1, 2, 7}; + ASSERT_EQ(filter.LookupField(fld.data(), fld.size()), ".n.x2.f4"); + + fld = {1, 2, 8, 5, 8, 5, 7}; + ASSERT_EQ(filter.LookupField(fld.data(), fld.size()), ".n.x2.c1.c2.c1.c2.f4"); +} + +} // namespace +} // namespace protozero diff --git a/src/protozero/filtering/message_filter.cc b/src/protozero/filtering/message_filter.cc new file mode 100644 index 000000000..a971dfab3 --- /dev/null +++ b/src/protozero/filtering/message_filter.cc @@ -0,0 +1,310 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/protozero/filtering/message_filter.h" + +#include "perfetto/base/logging.h" +#include "perfetto/protozero/proto_utils.h" + +namespace protozero { + +namespace { + +// Inline helpers to append proto fields in output. They are the equivalent of +// the protozero::Message::AppendXXX() fields but don't require building and +// maintaining a full protozero::Message object or dealing with scattered +// output slices. +// All these functions assume there is enough space in the output buffer, which +// should be always the case assuming that we don't end up generating more +// output than input. + +inline void AppendVarInt(uint32_t field_id, uint64_t value, uint8_t** out) { + *out = proto_utils::WriteVarInt(proto_utils::MakeTagVarInt(field_id), *out); + *out = proto_utils::WriteVarInt(value, *out); +} + +// For fixed32 / fixed64. +template <typename INT_T /* uint32_t | uint64_t*/> +inline void AppendFixed(uint32_t field_id, INT_T value, uint8_t** out) { + *out = proto_utils::WriteVarInt(proto_utils::MakeTagFixed<INT_T>(field_id), + *out); + memcpy(*out, &value, sizeof(value)); + *out += sizeof(value); +} + +// For length-delimited (string, bytes) fields. Note: this function appends only +// the proto preamble and the varint field that states the length of the payload +// not the payload itself. +// In the case of submessages, the caller needs to re-write the length at the +// end in the in the returned memory area. +// The problem here is that, because of filtering, the length of a submessage +// might be < original length (the original length is still an upper-bound). +// Returns a pair with: (1) the pointer where the final length should be written +// into, (2) the length of the size field. +// The caller must write a redundant varint to match the original size (i.e. +// needs to use WriteRedundantVarInt()). +inline std::pair<uint8_t*, uint32_t> AppendLenDelim(uint32_t field_id, + uint32_t len, + uint8_t** out) { + *out = proto_utils::WriteVarInt(proto_utils::MakeTagLengthDelimited(field_id), + *out); + uint8_t* size_field_start = *out; + *out = proto_utils::WriteVarInt(len, *out); + const size_t size_field_len = static_cast<size_t>(*out - size_field_start); + return std::make_pair(size_field_start, size_field_len); +} +} // namespace + +MessageFilter::MessageFilter() { + // Push a state on the stack for the implicit root message. + stack_.emplace_back(); +} + +MessageFilter::~MessageFilter() = default; + +bool MessageFilter::LoadFilterBytecode(const void* filter_data, size_t len) { + return filter_.Load(filter_data, len); +} + +bool MessageFilter::SetFilterRoot(const uint32_t* field_ids, + size_t num_fields) { + uint32_t root_msg_idx = 0; + for (const uint32_t* it = field_ids; it < field_ids + num_fields; ++it) { + uint32_t field_id = *it; + auto res = filter_.Query(root_msg_idx, field_id); + if (!res.allowed || res.simple_field()) + return false; + root_msg_idx = res.nested_msg_index; + } + root_msg_index_ = root_msg_idx; + return true; +} + +MessageFilter::FilteredMessage MessageFilter::FilterMessageFragments( + const InputSlice* slices, + size_t num_slices) { + // First compute the upper bound for the output. The filtered message cannot + // be > the original message. + uint32_t total_len = 0; + for (size_t i = 0; i < num_slices; ++i) + total_len += slices[i].len; + out_buf_.reset(new uint8_t[total_len]); + out_ = out_buf_.get(); + out_end_ = out_ + total_len; + + // Reset the parser state. + tokenizer_ = MessageTokenizer(); + error_ = false; + stack_.clear(); + stack_.resize(2); + // stack_[0] is a sentinel and should never be hit in nominal cases. If we + // end up there we will just keep consuming the input stream and detecting + // at the end, without hurting the fastpath. + stack_[0].in_bytes_limit = UINT32_MAX; + stack_[0].eat_next_bytes = UINT32_MAX; + // stack_[1] is the actual root message. + stack_[1].in_bytes_limit = total_len; + stack_[1].msg_index = root_msg_index_; + + // Process the input data and write the output. + for (size_t slice_idx = 0; slice_idx < num_slices; ++slice_idx) { + const InputSlice& slice = slices[slice_idx]; + const uint8_t* data = static_cast<const uint8_t*>(slice.data); + for (size_t i = 0; i < slice.len; ++i) + FilterOneByte(data[i]); + } + + // Construct the output object. + PERFETTO_CHECK(out_ >= out_buf_.get() && out_ <= out_end_); + auto used_size = static_cast<size_t>(out_ - out_buf_.get()); + FilteredMessage res{std::move(out_buf_), used_size}; + res.error = error_; + if (stack_.size() != 1 || !tokenizer_.idle() || + stack_[0].in_bytes != total_len) { + res.error = true; + } + return res; +} + +void MessageFilter::FilterOneByte(uint8_t octet) { + PERFETTO_DCHECK(!stack_.empty()); + + auto* state = &stack_.back(); + StackState next_state{}; + bool push_next_state = false; + + if (state->eat_next_bytes > 0) { + // This is the case where the previous tokenizer_.Push() call returned a + // length delimited message which is NOT a submessage (a string or a bytes + // field). We just want to consume it, and pass it through in output + // if the field was allowed. + --state->eat_next_bytes; + if (state->passthrough_eaten_bytes) + *(out_++) = octet; + } else { + MessageTokenizer::Token token = tokenizer_.Push(octet); + // |token| will not be valid() in most cases and this is WAI. When pushing + // a varint field, only the last byte yields a token, all the other bytes + // return an invalid token, they just update the internal tokenizer state. + if (token.valid()) { + auto filter = filter_.Query(state->msg_index, token.field_id); + switch (token.type) { + case proto_utils::ProtoWireType::kVarInt: + if (filter.allowed && filter.simple_field()) + AppendVarInt(token.field_id, token.value, &out_); + break; + case proto_utils::ProtoWireType::kFixed32: + if (filter.allowed && filter.simple_field()) + AppendFixed(token.field_id, static_cast<uint32_t>(token.value), + &out_); + break; + case proto_utils::ProtoWireType::kFixed64: + if (filter.allowed && filter.simple_field()) + AppendFixed(token.field_id, static_cast<uint64_t>(token.value), + &out_); + break; + case proto_utils::ProtoWireType::kLengthDelimited: + // Here we have two cases: + // A. A simple string/bytes field: we just want to consume the next + // bytes (the string payload), optionally passing them through in + // output if the field is allowed. + // B. This is a nested submessage. In this case we want to recurse and + // push a new state on the stack. + // Note that we can't tell the difference between a + // "non-allowed string" and a "non-allowed submessage". But it doesn't + // matter because in both cases we just want to skip the next N bytes. + const auto submessage_len = static_cast<uint32_t>(token.value); + auto in_bytes_left = state->in_bytes_limit - state->in_bytes - 1; + if (PERFETTO_UNLIKELY(submessage_len > in_bytes_left)) { + // This is a malicious / malformed string/bytes/submessage that + // claims to be larger than the outer message that contains it. + return SetUnrecoverableErrorState(); + } + + if (filter.allowed && !filter.simple_field() && submessage_len > 0) { + // submessage_len == 0 is the edge case of a message with a 0-len + // (but present) submessage. In this case, if allowed, we don't want + // to push any further state (doing so would desync the FSM) but we + // still want to emit it. + // At this point |submessage_len| is only an upper bound. The + // final message written in output can be <= the one in input, + // only some of its fields might be allowed (also remember that + // this class implicitly removes redundancy varint encoding of + // len-delimited field lengths). The final length varint (the + // return value of AppendLenDelim()) will be filled when popping + // from |stack_|. + auto size_field = + AppendLenDelim(token.field_id, submessage_len, &out_); + push_next_state = true; + next_state.field_id = token.field_id; + next_state.msg_index = filter.nested_msg_index; + next_state.in_bytes_limit = submessage_len; + next_state.size_field = size_field.first; + next_state.size_field_len = size_field.second; + next_state.out_bytes_written_at_start = out_written(); + } else { + // A string or bytes field, or a 0 length submessage. + state->eat_next_bytes = submessage_len; + state->passthrough_eaten_bytes = filter.allowed; + if (filter.allowed) + AppendLenDelim(token.field_id, submessage_len, &out_); + } + break; + } // switch(type) + + if (PERFETTO_UNLIKELY(track_field_usage_)) { + IncrementCurrentFieldUsage(token.field_id, filter.allowed); + } + } // if (token.valid) + } // if (eat_next_bytes == 0) + + ++state->in_bytes; + while (state->in_bytes >= state->in_bytes_limit) { + PERFETTO_DCHECK(state->in_bytes == state->in_bytes_limit); + push_next_state = false; + + // We can't possibly write more than we read. + const uint32_t msg_bytes_written = static_cast<uint32_t>( + out_written() - state->out_bytes_written_at_start); + PERFETTO_DCHECK(msg_bytes_written <= state->in_bytes_limit); + + // Backfill the length field of the + proto_utils::WriteRedundantVarInt(msg_bytes_written, state->size_field, + state->size_field_len); + + const uint32_t in_bytes_processes_for_last_msg = state->in_bytes; + stack_.pop_back(); + PERFETTO_CHECK(!stack_.empty()); + state = &stack_.back(); + state->in_bytes += in_bytes_processes_for_last_msg; + if (PERFETTO_UNLIKELY(!tokenizer_.idle())) { + // If we hit this case, it means that we got to the end of a submessage + // while decoding a field. We can't recover from this and we don't want to + // propagate a broken sub-message. + return SetUnrecoverableErrorState(); + } + } + + if (push_next_state) { + PERFETTO_DCHECK(tokenizer_.idle()); + stack_.emplace_back(std::move(next_state)); + state = &stack_.back(); + } +} + +void MessageFilter::SetUnrecoverableErrorState() { + error_ = true; + stack_.clear(); + stack_.resize(1); + auto& state = stack_[0]; + state.eat_next_bytes = UINT32_MAX; + state.in_bytes_limit = UINT32_MAX; + state.passthrough_eaten_bytes = false; + out_ = out_buf_.get(); // Reset the write pointer. +} + +void MessageFilter::IncrementCurrentFieldUsage(uint32_t field_id, + bool allowed) { + // Slowpath. Used mainly in offline tools and tests to workout used fields in + // a proto. + PERFETTO_DCHECK(track_field_usage_); + + // Field path contains a concatenation of varints, one for each nesting level. + // e.g. y in message Root { Sub x = 2; }; message Sub { SubSub y = 7; } + // is encoded as [varint(2) + varint(7)]. + // We use varint to take the most out of SSO (small string opt). In most cases + // the path will fit in the on-stack 22 bytes, requiring no heap. + std::string field_path; + + auto append_field_id = [&field_path](uint32_t id) { + uint8_t buf[10]; + uint8_t* end = proto_utils::WriteVarInt(id, buf); + field_path.append(reinterpret_cast<char*>(buf), + static_cast<size_t>(end - buf)); + }; + + // Append all the ancestors IDs from the state stack. + // The first entry of the stack has always ID 0 and we skip it (we don't know + // the ID of the root message itself). + PERFETTO_DCHECK(stack_.size() >= 2 && stack_[1].field_id == 0); + for (size_t i = 2; i < stack_.size(); ++i) + append_field_id(stack_[i].field_id); + // Append the id of the field in the current message. + append_field_id(field_id); + field_usage_[field_path] += allowed ? 1 : -1; +} + +} // namespace protozero diff --git a/src/protozero/filtering/message_filter.h b/src/protozero/filtering/message_filter.h new file mode 100644 index 000000000..37dc9b520 --- /dev/null +++ b/src/protozero/filtering/message_filter.h @@ -0,0 +1,208 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_PROTOZERO_FILTERING_MESSAGE_FILTER_H_ +#define SRC_PROTOZERO_FILTERING_MESSAGE_FILTER_H_ + +#include <stdint.h> + +#include <memory> +#include <string> +#include <unordered_map> + +#include "src/protozero/filtering/filter_bytecode_parser.h" +#include "src/protozero/filtering/message_tokenizer.h" + +namespace protozero { + +// A class to filter binary-encoded proto messages using an allow-list of field +// ids, also known as "filter bytecode". The filter determines which fields are +// allowed to be passed through in output and strips all the other fields. +// See go/trace-filtering for full design. +// This class takes in input: +// 1) The filter bytecode, loaded once via the LoadFilterBytecode() method. +// 2) A proto-encoded binary message. The message doesn't have to be contiguous, +// it can be passed as an array of arbitrarily chunked fragments. +// The FilterMessage*() method returns in output a proto message, stripping out +// all unknown fields. If the input is malformed (e.g., unknown proto field wire +// types, lengths out of bound) the whole filtering failed and the |error| flag +// of the FilteredMessage object is set to true. +// The filtering operation is based on rewriting a copy of the message into a +// self-allocated buffer, which is then returned in the output. The input buffer +// is NOT altered. +// Note also that the process of rewriting the protos gets rid of most redundant +// varint encoding (if present). So even if all fields are allow-listed, the +// output might NOT be bitwise identical to the input (but it will be +// semantically equivalent). +// Furthermore the enable_field_usage_tracking() method allows to keep track of +// a histogram of allowed / denied fields. It slows down filtering and is +// intended only on host tools. +class MessageFilter { + public: + MessageFilter(); + ~MessageFilter(); + + struct InputSlice { + const void* data; + size_t len; + }; + + struct FilteredMessage { + FilteredMessage(std::unique_ptr<uint8_t[]> d, size_t s) + : data(std::move(d)), size(s) {} + std::unique_ptr<uint8_t[]> data; + size_t size; // The used bytes in |data|. This is <= sizeof(data). + bool error = false; + }; + + // Loads the filter bytecode that will be used to filter any subsequent + // message. Must be called before the first call to FilterMessage*(). + // |filter_data| must point to a byte buffer for a proto-encoded ProtoFilter + // message (see proto_filter.proto). + bool LoadFilterBytecode(const void* filter_data, size_t len); + + // This affects the filter starting point of the subsequent FilterMessage*() + // calls. By default the filtering process starts from the message @ index 0, + // the root message passed to proto_filter when generating the bytecode + // (in typical tracing use-cases, this is perfetto.protos.Trace). However, the + // caller (TracingServiceImpl) might want to filter packets from the 2nd level + // (perfetto.protos.TracePacket) because the root level is pre-pended after + // the fact. This call allows to change the root message for the filter. + // The argument |field_ids| is an array of proto field ids and determines the + // path to the new root. For instance, in the case of [1,2,3] SetFilterRoot + // will identify the sub-message for the field "root.1.2.3" and use that. + // In order for this to succeed all the fields in the path must be allowed + // in the filter and must be a nested message type. + bool SetFilterRoot(const uint32_t* field_ids, size_t num_fields); + + // Takes an input message, fragmented in arbitrary slices, and returns a + // filtered message in output. + FilteredMessage FilterMessageFragments(const InputSlice*, size_t num_slices); + + // Helper for tests, where the input is a contiguous buffer. + FilteredMessage FilterMessage(const void* data, size_t len) { + InputSlice slice{data, len}; + return FilterMessageFragments(&slice, 1); + } + + // When enabled returns a map of "field path" to "usage counter". + // The key (std::string) is a binary buffer (i.e. NOT an ASCII/UTF-8 string) + // which contains a varint for each field. Consider the following: + // message Root { Sub1 f1 = 1; }; + // message Sub1 { Sub2 f2 = 7;} + // message Sub2 { string f3 = 5; } + // The field .f1.f2.f3 will be encoded as \x01\0x07\x05. + // The value is the number of times that field has been encountered. If the + // field is not allow-listed in the bytecode (the field is stripped in output) + // the count will be negative. + void enable_field_usage_tracking(bool x) { track_field_usage_ = x; } + const std::unordered_map<std::string, int32_t>& field_usage() const { + return field_usage_; + } + + // Exposed only for DCHECKS in TracingServiceImpl. + uint32_t root_msg_index() { return root_msg_index_; } + + private: + // This is called by FilterMessageFragments(). + // Inlining allows the compiler turn the per-byte call/return into a for loop, + // while, at the same time, keeping the code easy to read and reason about. + // It gives a 20-25% speedup (265ms vs 215ms for a 25MB trace). + void FilterOneByte(uint8_t octet) PERFETTO_ALWAYS_INLINE; + + // No-inline because this is a slowpath (only when usage tracking is enabled). + void IncrementCurrentFieldUsage(uint32_t field_id, + bool allowed) PERFETTO_NO_INLINE; + + // Gets into an error state which swallows all the input and emits no output. + void SetUnrecoverableErrorState(); + + // We keep track of the the nest of messages in a stack. Each StackState + // object corresponds to a level of nesting in the proto message structure. + // Every time a new field of type len-delimited that has a corresponding + // sub-message in the bytecode is encountered, a new StackState is pushed in + // |stack_|. stack_[0] is a sentinel to prevent over-popping without adding + // extra branches in the fastpath. + // |stack_|. stack_[1] is the state of the root message. + struct StackState { + uint32_t in_bytes = 0; // Number of input bytes processed. + + // When |in_bytes| reaches this value, the current state should be popped. + // This is set when recursing into nested submessages. This is 0 only for + // stack_[0] (we don't know the size of the root message upfront). + uint32_t in_bytes_limit = 0; + + // This is set when a len-delimited message is encountered, either a string + // or a nested submessage that is NOT allow-listed in the bytecode. + // This causes input bytes to be consumed without being parsed from the + // input stream. If |passthrough_eaten_bytes| == true, they will be copied + // as-is in output (e.g. in the case of an allowed string/bytes field). + uint32_t eat_next_bytes = 0; + + // Keeps tracks of the stream_writer output counter (out_.written()) then + // the StackState is pushed. This is used to work out, when popping, how + // many bytes have been written for the current submessage. + uint32_t out_bytes_written_at_start = 0; + + uint32_t field_id = 0; // The proto field id for the current message. + uint32_t msg_index = 0; // The index of the message filter in the bytecode. + + // This is a pointer to the proto preamble for the current submessage + // (it's nullptr for stack_[0] and non-null elsewhere). This will be filled + // with the actual size of the message (out_.written() - + // |out_bytes_written_at_start|) when finishing (popping) the message. + // This must be filled using WriteRedundantVarint(). Note that the + // |size_field_len| is variable and depends on the actual length of the + // input message. If the output message has roughly the same size of the + // input message, the length will not be redundant. + // In other words: the length of the field is reserved when the submessage + // starts. At that point we know the upper-bound for the output message + // (a filtered submessage can be <= the original one, but not >). So we + // reserve as many bytes it takes to write the input length in varint. + // Then, when the message is finalized and we know the actual output size + // we backfill the field. + // Consider the example of a submessage where the input size = 130 (>127, + // 2 varint bytes) and the output is 120 bytes. The length will be 2 bytes + // wide even though could have been encoded with just one byte. + uint8_t* size_field = nullptr; + uint32_t size_field_len = 0; + + // When true the next |eat_next_bytes| are copied as-is in output. + // It seems that keeping this field at the end rather than next to + // |eat_next_bytes| makes the filter a little (but measurably) faster. + // (likely something related with struct layout vs cache sizes). + bool passthrough_eaten_bytes = false; + }; + + uint32_t out_written() { return static_cast<uint32_t>(out_ - &out_buf_[0]); } + + std::unique_ptr<uint8_t[]> out_buf_; + uint8_t* out_ = nullptr; + uint8_t* out_end_ = nullptr; + uint32_t root_msg_index_ = 0; + + FilterBytecodeParser filter_; + MessageTokenizer tokenizer_; + std::vector<StackState> stack_; + + bool error_ = false; + bool track_field_usage_ = false; + std::unordered_map<std::string, int32_t> field_usage_; +}; + +} // namespace protozero + +#endif // SRC_PROTOZERO_FILTERING_MESSAGE_FILTER_H_ diff --git a/src/protozero/filtering/message_filter_benchmark.cc b/src/protozero/filtering/message_filter_benchmark.cc new file mode 100644 index 000000000..c800c8234 --- /dev/null +++ b/src/protozero/filtering/message_filter_benchmark.cc @@ -0,0 +1,48 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <benchmark/benchmark.h> + +#include <algorithm> +#include <string> + +#include "perfetto/ext/base/file_utils.h" +#include "src/base/test/utils.h" +#include "src/protozero/filtering/message_filter.h" + +static void BM_ProtozeroMessageFilter(benchmark::State& state) { + std::string trace_data; + static const char kTestTrace[] = "test/data/example_android_trace_30s.pb"; + perfetto::base::ReadFile(perfetto::base::GetTestDataPath(kTestTrace), + &trace_data); + PERFETTO_CHECK(!trace_data.empty()); + + std::string filter; + static const char kFullTraceFilter[] = "test/data/full_trace_filter.bytecode"; + perfetto::base::ReadFile(kFullTraceFilter, &filter); + PERFETTO_CHECK(!filter.empty()); + + protozero::MessageFilter filt; + filt.LoadFilterBytecode(filter.data(), filter.size()); + + for (auto _ : state) { + auto res = filt.FilterMessage(trace_data.data(), trace_data.size()); + benchmark::DoNotOptimize(res); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed( + static_cast<int64_t>(state.iterations() * trace_data.size())); +} + +BENCHMARK(BM_ProtozeroMessageFilter); diff --git a/src/protozero/filtering/message_filter_fuzzer.cc b/src/protozero/filtering/message_filter_fuzzer.cc new file mode 100644 index 000000000..60cfe5c84 --- /dev/null +++ b/src/protozero/filtering/message_filter_fuzzer.cc @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#include "perfetto/base/logging.h" +#include "src/protozero/filtering/message_filter.h" + +namespace protozero { +namespace { + +// A valid filter bytecode obtained from a perfetto.protos.Trace message. +uint8_t kValidFilter[] = { + 0x0b, 0x01, 0x12, 0x04, 0x00, 0x0b, 0x02, 0x13, 0x0f, 0x19, 0x23, 0x13, + 0x33, 0x14, 0x3b, 0x15, 0x41, 0x4b, 0x11, 0x51, 0x5b, 0x16, 0x63, 0x3b, + 0x69, 0x8b, 0x02, 0x21, 0x93, 0x02, 0x2a, 0x9b, 0x02, 0x2c, 0xab, 0x02, + 0x2e, 0xb3, 0x02, 0x09, 0xc3, 0x02, 0x30, 0xca, 0x02, 0x02, 0xdb, 0x02, + 0x31, 0xe3, 0x02, 0x26, 0xeb, 0x02, 0x32, 0xf3, 0x02, 0x07, 0xfb, 0x02, + 0x33, 0x8b, 0x03, 0x0a, 0x9b, 0x03, 0x34, 0xb3, 0x03, 0x06, 0xc3, 0x03, + 0x37, 0xd1, 0x03, 0xdb, 0x03, 0x3c, 0xe3, 0x03, 0x39, 0x93, 0x04, 0x38, + 0x9b, 0x04, 0x3a, 0x00, 0x09, 0x13, 0x03, 0x19, 0x00, 0x0a, 0x02, 0x1b, + 0x04, 0x23, 0x05, 0x5b, 0x06, 0x6b, 0x06, 0x83, 0x01, 0x07, 0x8b, 0x01, + 0x08, 0x93, 0x01, 0x07, 0x9b, 0x01, 0x07, 0xa3, 0x01, 0x08, 0x9b, 0x02, + 0x08, 0xcb, 0x02, 0x08, 0xd3, 0x02, 0x08, 0xdb, 0x02, 0x09, 0xe3, 0x02, + 0x07, 0xeb, 0x02, 0x0a, 0xf3, 0x02, 0x07, 0xfb, 0x02, 0x0b, 0x83, 0x03, + 0x06, 0x8b, 0x03, 0x0b, 0x93, 0x03, 0x05, 0x9b, 0x03, 0x0b, 0xab, 0x03, + 0x0c, 0xb3, 0x03, 0x0c, 0xbb, 0x03, 0x0c, 0x9b, 0x04, 0x0d, 0xa3, 0x04, + 0x07, 0xab, 0x04, 0x06, 0xb3, 0x04, 0x07, 0xc3, 0x04, 0x06, 0xcb, 0x04, + 0x07, 0xd3, 0x04, 0x07, 0xdb, 0x04, 0x06, 0x93, 0x07, 0x08, 0xeb, 0x07, + 0x08, 0xcb, 0x09, 0x07, 0xd3, 0x09, 0x05, 0xf3, 0x0b, 0x06, 0xdb, 0x0e, + 0x09, 0xe3, 0x0e, 0x09, 0xeb, 0x0e, 0x07, 0xf3, 0x0e, 0x09, 0xfb, 0x0e, + 0x09, 0x83, 0x0f, 0x07, 0x8b, 0x0f, 0x06, 0x93, 0x0f, 0x07, 0xb3, 0x0f, + 0x0a, 0xc3, 0x0f, 0x0e, 0xfb, 0x0f, 0x0e, 0x83, 0x10, 0x08, 0xfb, 0x10, + 0x08, 0x8b, 0x11, 0x08, 0xcb, 0x13, 0x06, 0xd3, 0x13, 0x07, 0xdb, 0x13, + 0x07, 0xb3, 0x14, 0x07, 0x00, 0x11, 0x00, 0x0a, 0x07, 0x00, 0x0a, 0x02, + 0x00, 0x0a, 0x03, 0x00, 0x0a, 0x05, 0x00, 0x0a, 0x04, 0x00, 0x0a, 0x06, + 0x00, 0x09, 0x00, 0x00, 0x0a, 0x03, 0x29, 0x00, 0x0a, 0x08, 0x00, 0x0b, + 0x10, 0x13, 0x07, 0x19, 0x00, 0x0a, 0x03, 0x23, 0x07, 0x29, 0x00, 0x0b, + 0x12, 0x11, 0x00, 0x0a, 0x0a, 0x5b, 0x07, 0x00, 0x0a, 0x02, 0x1b, 0x07, + 0x00, 0x0b, 0x09, 0x11, 0x00, 0x0b, 0x06, 0x13, 0x06, 0x1b, 0x0e, 0x22, + 0x02, 0x33, 0x06, 0x39, 0x43, 0x06, 0x49, 0x00, 0x0a, 0x03, 0x2b, 0x0b, + 0x33, 0x20, 0x42, 0x05, 0x82, 0x01, 0x02, 0xa1, 0x01, 0xc3, 0x01, 0x17, + 0xcb, 0x01, 0x04, 0xd3, 0x01, 0x0b, 0xdb, 0x01, 0x06, 0xe3, 0x01, 0x1e, + 0xeb, 0x01, 0x1f, 0xf2, 0x01, 0x02, 0x00, 0x0b, 0x18, 0x12, 0x0c, 0x73, + 0x1a, 0x7b, 0x1c, 0x83, 0x01, 0x1d, 0x8b, 0x01, 0x05, 0x00, 0x0b, 0x08, + 0x13, 0x19, 0x00, 0x0a, 0x2e, 0x00, 0x0a, 0x03, 0x23, 0x1b, 0x2b, 0x1b, + 0x33, 0x05, 0x00, 0x0a, 0x09, 0x53, 0x09, 0x00, 0x09, 0x13, 0x1b, 0x00, + 0x0a, 0x03, 0x23, 0x1b, 0x00, 0x09, 0x19, 0x00, 0x0a, 0x03, 0x23, 0x06, + 0x2a, 0x02, 0x00, 0x0a, 0x04, 0x31, 0x42, 0x08, 0x92, 0x01, 0x02, 0x00, + 0x0b, 0x22, 0x13, 0x23, 0x1a, 0x03, 0x33, 0x07, 0x3b, 0x09, 0x42, 0x03, + 0x5b, 0x0b, 0x62, 0x03, 0x81, 0x01, 0x8b, 0x01, 0x29, 0x92, 0x01, 0x02, + 0xa3, 0x01, 0x07, 0xab, 0x01, 0x0b, 0xb2, 0x01, 0x03, 0xcb, 0x01, 0x09, + 0xda, 0x01, 0x02, 0x00, 0x09, 0x21, 0x00, 0x0b, 0x24, 0x11, 0x00, 0x0a, + 0x04, 0x31, 0xa3, 0x06, 0x25, 0xbb, 0x06, 0x26, 0xc3, 0x06, 0x0a, 0xcb, + 0x06, 0x27, 0xd3, 0x06, 0x06, 0xeb, 0x06, 0x0b, 0x00, 0x0a, 0x03, 0x52, + 0x02, 0x00, 0x0a, 0x04, 0x32, 0x03, 0x00, 0x0a, 0x02, 0x22, 0x02, 0x33, + 0x28, 0x3a, 0x02, 0x00, 0x2a, 0x02, 0x00, 0x09, 0x13, 0x07, 0x19, 0x00, + 0x09, 0x13, 0x2b, 0x00, 0x0a, 0x09, 0x00, 0x0b, 0x2d, 0x12, 0x08, 0x00, + 0x0a, 0x12, 0x00, 0x0b, 0x06, 0x13, 0x09, 0x1b, 0x06, 0x23, 0x05, 0x2b, + 0x2f, 0x32, 0x02, 0x00, 0x09, 0x13, 0x0a, 0x00, 0x0b, 0x09, 0x13, 0x07, + 0x00, 0x09, 0x1a, 0x03, 0x00, 0x0b, 0x09, 0x12, 0x02, 0x00, 0x0b, 0x08, + 0x12, 0x02, 0x00, 0x0b, 0x35, 0x11, 0x00, 0x0b, 0x36, 0x13, 0x36, 0x00, + 0x09, 0x13, 0x07, 0x1b, 0x0b, 0x00, 0x09, 0x13, 0x08, 0x1b, 0x06, 0x23, + 0x06, 0x2a, 0x02, 0x3b, 0x06, 0x00, 0x0a, 0x05, 0x82, 0x01, 0x03, 0x00, + 0x09, 0x1b, 0x31, 0x23, 0x26, 0x29, 0x33, 0x09, 0x3b, 0x06, 0x43, 0x31, + 0x00, 0x0b, 0x06, 0x00, 0x0b, 0x06, 0x13, 0x06, 0x23, 0x09, 0x2b, 0x06, + 0x33, 0x09, 0x3b, 0x06, 0x83, 0x01, 0x06, 0x8b, 0x01, 0x06, 0x93, 0x01, + 0x06, 0x9b, 0x01, 0x05, 0x00, 0x5b, 0x3d, 0xd1, 0x03, 0x00, 0x59, 0xf9, + 0x01, 0x00, 0x8f, 0xf8, 0xf5, 0xcb, 0x06}; + +int FuzzMessageFilter(const uint8_t* data, size_t size) { + MessageFilter filter; + PERFETTO_CHECK(filter.LoadFilterBytecode(kValidFilter, sizeof(kValidFilter))); + + auto res = filter.FilterMessage(data, size); + + // Either parsing fails or if it succeeds, the output data must be <= input. + PERFETTO_CHECK(res.error || res.size <= size); + return 0; +} + +} // namespace +} // namespace protozero + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size); + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + return protozero::FuzzMessageFilter(data, size); +} diff --git a/src/protozero/filtering/message_filter_unittest.cc b/src/protozero/filtering/message_filter_unittest.cc new file mode 100644 index 000000000..83cb911f2 --- /dev/null +++ b/src/protozero/filtering/message_filter_unittest.cc @@ -0,0 +1,810 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "test/gtest_and_gmock.h" + +#include <random> + +#include "perfetto/ext/base/file_utils.h" +#include "perfetto/ext/base/string_utils.h" +#include "perfetto/ext/base/temp_file.h" +#include "perfetto/protozero/proto_decoder.h" +#include "perfetto/protozero/proto_utils.h" +#include "perfetto/protozero/scattered_heap_buffer.h" +#include "protos/perfetto/trace/trace.pb.h" +#include "src/protozero/filtering/filter_util.h" +#include "src/protozero/filtering/message_filter.h" + +namespace protozero { + +namespace { + +TEST(MessageFilterTest, EndToEnd) { + auto schema = perfetto::base::TempFile::Create(); + static const char kSchema[] = R"( + syntax = "proto2"; + message FilterSchema { + message Nested { + optional fixed32 f32 = 2; + repeated string ss = 5; + } + optional int32 i32 = 1; + optional string str = 3; + repeated Nested nest = 6; + repeated int32 f11 = 11; + repeated int64 f12 = 12; + repeated sint32 f13 = 13; + repeated sint64 f14 = 14; + repeated fixed32 f15 = 15; + repeated fixed32 f16 = 16; + repeated fixed64 f17 = 17; + repeated fixed64 f18 = 18; + repeated float f19 = 19; + repeated double f20 = 20; + }; + )"; + + perfetto::base::WriteAll(*schema, kSchema, strlen(kSchema)); + perfetto::base::FlushFile(*schema); + + FilterUtil filter; + ASSERT_TRUE(filter.LoadMessageDefinition(schema.path(), "", "")); + std::string bytecode = filter.GenerateFilterBytecode(); + ASSERT_GT(bytecode.size(), 0u); + + HeapBuffered<Message> msg; + msg->AppendVarInt(/*field_id=*/1, -1000000000ll); + msg->AppendVarInt(/*field_id=*/2, 42); + msg->AppendString(/*field_id=*/3, "foobar"); + msg->AppendFixed(/*field_id=*/4, 10); + msg->AppendVarInt(/*field_id=*/11, INT32_MIN); + msg->AppendVarInt(/*field_id=*/12, INT64_MIN); + msg->AppendSignedVarInt(/*field_id=*/13, INT32_MIN); + msg->AppendSignedVarInt(/*field_id=*/14, INT64_MIN); + msg->AppendFixed(/*field_id=*/15, static_cast<int32_t>(INT32_MIN)); + msg->AppendFixed(/*field_id=*/16, static_cast<int32_t>(INT32_MAX)); + msg->AppendFixed(/*field_id=*/17, static_cast<int64_t>(INT64_MIN)); + msg->AppendFixed(/*field_id=*/18, static_cast<int64_t>(INT64_MAX)); + msg->AppendFixed(/*field_id=*/19, FLT_EPSILON); + msg->AppendFixed(/*field_id=*/20, DBL_EPSILON); + + auto* nest = msg->BeginNestedMessage<Message>(/*field_id=*/6); + nest->AppendFixed(/*field_id=*/1, 10); + nest->AppendFixed(/*field_id=*/2, static_cast<int32_t>(-2000000000ll)); + nest->AppendString(/*field_id=*/4, "stripped"); + nest->AppendString(/*field_id=*/5, ""); + nest->Finalize(); + + MessageFilter flt; + ASSERT_TRUE(flt.LoadFilterBytecode(bytecode.data(), bytecode.size())); + + std::vector<uint8_t> encoded = msg.SerializeAsArray(); + + for (int repetitions = 0; repetitions < 3; ++repetitions) { + auto filtered = flt.FilterMessage(encoded.data(), encoded.size()); + ASSERT_LT(filtered.size, encoded.size()); + + ProtoDecoder dec(filtered.data.get(), filtered.size); + EXPECT_TRUE(dec.FindField(1).valid()); + EXPECT_EQ(dec.FindField(1).as_int64(), -1000000000ll); + EXPECT_FALSE(dec.FindField(2).valid()); + EXPECT_TRUE(dec.FindField(3).valid()); + EXPECT_EQ(dec.FindField(3).as_std_string(), "foobar"); + EXPECT_FALSE(dec.FindField(4).valid()); + EXPECT_TRUE(dec.FindField(6).valid()); + for (uint32_t i = 11; i <= 20; ++i) + EXPECT_TRUE(dec.FindField(i).valid()); + + EXPECT_EQ(dec.FindField(11).as_int32(), INT32_MIN); + EXPECT_EQ(dec.FindField(12).as_int64(), INT64_MIN); + EXPECT_EQ(dec.FindField(13).as_sint32(), INT32_MIN); + EXPECT_EQ(dec.FindField(14).as_sint64(), INT64_MIN); + EXPECT_EQ(dec.FindField(15).as_int32(), INT32_MIN); + EXPECT_EQ(dec.FindField(16).as_int32(), INT32_MAX); + EXPECT_EQ(dec.FindField(17).as_int64(), INT64_MIN); + EXPECT_EQ(dec.FindField(18).as_int64(), INT64_MAX); + EXPECT_EQ(dec.FindField(19).as_float(), FLT_EPSILON); + EXPECT_EQ(dec.FindField(20).as_double(), DBL_EPSILON); + + ProtoDecoder nest_dec(dec.FindField(6).as_bytes()); + EXPECT_FALSE(nest_dec.FindField(1).valid()); + EXPECT_TRUE(nest_dec.FindField(2).valid()); + EXPECT_EQ(nest_dec.FindField(2).as_int32(), -2000000000ll); + EXPECT_TRUE(nest_dec.FindField(5).valid()); + EXPECT_EQ(nest_dec.FindField(5).as_bytes().size, 0u); + } +} + +TEST(MessageFilterTest, ChangeRoot) { + auto schema = perfetto::base::TempFile::Create(); + static const char kSchema[] = R"( + syntax = "proto2"; + message FilterSchema { + message Nested { + message Nested2 { + optional int32 e = 5; + } + optional int32 c = 3; + repeated Nested2 d = 4; + } + optional int32 a = 1; + optional Nested b = 2; + }; + )"; + + perfetto::base::WriteAll(*schema, kSchema, strlen(kSchema)); + perfetto::base::FlushFile(*schema); + + FilterUtil filter; + ASSERT_TRUE(filter.LoadMessageDefinition(schema.path(), "", "")); + std::string bytecode = filter.GenerateFilterBytecode(); + ASSERT_GT(bytecode.size(), 0u); + + HeapBuffered<Message> msg; + msg->AppendVarInt(/*field_id=*/1, 101); + msg->AppendVarInt(/*field_id=*/3, 103); + msg->AppendVarInt(/*field_id=*/5, 105); + auto* nest = msg->BeginNestedMessage<Message>(/*field_id=*/4); // Nested b. + nest->AppendVarInt(/*field_id=*/5, 205); + nest->Finalize(); + std::vector<uint8_t> encoded = msg.SerializeAsArray(); + + MessageFilter flt; + ASSERT_TRUE(flt.LoadFilterBytecode(bytecode.data(), bytecode.size())); + uint32_t roots[2]{2, 4}; + + // First set the root to field id ".2" (.b). The fliter should happen treating + // |Nested| as rot, so allowing only field 3 and 4 (Nested2) through. + { + flt.SetFilterRoot(roots, 1); + auto filtered = flt.FilterMessage(encoded.data(), encoded.size()); + ASSERT_LT(filtered.size, encoded.size()); + ProtoDecoder dec(filtered.data.get(), filtered.size); + EXPECT_FALSE(dec.FindField(1).valid()); + EXPECT_TRUE(dec.FindField(3).valid()); + EXPECT_EQ(dec.FindField(3).as_int32(), 103); + EXPECT_FALSE(dec.FindField(5).valid()); + EXPECT_TRUE(dec.FindField(4).valid()); + EXPECT_EQ(dec.FindField(4).as_std_string(), "(\xCD\x01"); + } + + // Now set the root to ".2.4" (.b.d). This should allow only the field "e" + // to pass through. + { + flt.SetFilterRoot(roots, 2); + auto filtered = flt.FilterMessage(encoded.data(), encoded.size()); + ASSERT_LT(filtered.size, encoded.size()); + ProtoDecoder dec(filtered.data.get(), filtered.size); + EXPECT_FALSE(dec.FindField(1).valid()); + EXPECT_FALSE(dec.FindField(3).valid()); + EXPECT_FALSE(dec.FindField(4).valid()); + EXPECT_TRUE(dec.FindField(5).valid()); + EXPECT_EQ(dec.FindField(5).as_int32(), 105); + } +} + +TEST(MessageFilterTest, MalformedInput) { + // Create and load a simple filter. + auto schema = perfetto::base::TempFile::Create(); + static const char kSchema[] = R"( + syntax = "proto2"; + message FilterSchema { + message Nested { + optional fixed32 f32 = 4; + repeated string ss = 5; + } + optional int32 i32 = 1; + optional string str = 2; + repeated Nested nest = 3; + }; + )"; + perfetto::base::WriteAll(*schema, kSchema, strlen(kSchema)); + perfetto::base::FlushFile(*schema); + FilterUtil filter; + ASSERT_TRUE(filter.LoadMessageDefinition(schema.path(), "", "")); + std::string bytecode = filter.GenerateFilterBytecode(); + ASSERT_GT(bytecode.size(), 0u); + MessageFilter flt; + ASSERT_TRUE(flt.LoadFilterBytecode(bytecode.data(), bytecode.size())); + + { + // A malformed message found by the fuzzer. + static const uint8_t kData[]{ + 0x52, 0x21, // ID=10, type=len-delimited, len=33. + 0xa0, 0xa4, // Early terminating payload. + }; + auto res = flt.FilterMessage(kData, sizeof(kData)); + EXPECT_TRUE(res.error); + } + + { + // A malformed message which contains a non-terminated varint. + static const uint8_t kData[]{ + 0x08, 0x2A, // A valid varint field id=1 value=42 (0x2A). + 0x08, 0xFF, // An unterminated varint. + }; + auto res = flt.FilterMessage(kData, sizeof(kData)); + EXPECT_TRUE(res.error); + } + + { + // A malformed message which contains a sub-message with a field that brings + // it out of the outer size. + static const uint8_t kData[]{ + 0x08, 0x2A, // A valid varint field id=1 value=42 (0x2A). + 0x1A, 0x04, // A len-delim field, id=3, length=4. + // The nested message |nest| starts here. + 0x25, 0x0, 0x0, 0x0, 0x01, // A fixed32 field, id=4. + // Note that the fixed32 field has an expected length of 4 but that + // overflows the size of the |nest| method, because with its 0x25 + // preamble it becomes 5 bytes. At this point this should cause a + // persistent failure. + }; + auto res = flt.FilterMessage(kData, sizeof(kData)); + EXPECT_TRUE(res.error); + } + + // A parsing failure shoulnd't affect the ability to filter the following + // message. Try again but this time with a valid message. + { + static const uint8_t kData[]{ + 0x08, 0x2A, // A valid varint field id=1 value=42 (0x2A). + 0x1A, 0x05, // A len-delim field, id=3, length=5. + 0x25, 0x0, 0x0, 0x0, 0x01, // A fixed32 field, id=4. + 0x38, 0x42, // A valid but not allowed varint field id=7. + }; + auto res = flt.FilterMessage(kData, sizeof(kData)); + EXPECT_FALSE(res.error); + EXPECT_EQ(res.size, sizeof(kData) - 2); // last 2 bytes should be skipped. + EXPECT_EQ(memcmp(kData, res.data.get(), res.size), 0); + } +} + +// It processes a real test trace with a real filter. The filter has been +// obtained from the full upstream perfetto proto (+ re-adding the for_testing +// field which got removed after adding most test traces). This covers the most +// complex case of filtering a real trace with a filter that allows all possible +// fields, hence re-entering deeply in most nested fields. +TEST(MessageFilterTest, RealTracePassthrough) { + // This is test/data/android_log_ring_buffer_mode.pb. It's re-encoded as a + // constant because unittests cannot depend on test/data/, only integration + // tests can. + static const uint8_t kTraceData[]{ + 0x0a, 0x16, 0x18, 0x8f, 0x4e, 0xa2, 0x02, 0x10, 0x82, 0x47, 0x7a, 0x76, + 0xb2, 0x8d, 0x42, 0xba, 0x81, 0xdc, 0x33, 0x32, 0x6d, 0x57, 0xa0, 0x79, + 0x0a, 0x5f, 0x18, 0x8f, 0x4e, 0x32, 0x5a, 0x0a, 0x09, 0x08, 0x06, 0x10, + 0xf4, 0xd3, 0xea, 0xbb, 0xba, 0x55, 0x0a, 0x0c, 0x08, 0x02, 0x10, 0xf9, + 0xcc, 0xb4, 0xd1, 0xe8, 0xdc, 0xa5, 0xbc, 0x15, 0x0a, 0x09, 0x08, 0x04, + 0x10, 0x86, 0xb9, 0x9c, 0xba, 0xba, 0x55, 0x0a, 0x0c, 0x08, 0x01, 0x10, + 0xeb, 0xe9, 0x82, 0xd3, 0xe8, 0xdc, 0xa5, 0xbc, 0x15, 0x0a, 0x09, 0x08, + 0x03, 0x10, 0xac, 0xd6, 0xea, 0xbb, 0xba, 0x55, 0x0a, 0x09, 0x08, 0x05, + 0x10, 0x9b, 0xe1, 0xd8, 0xbb, 0xba, 0x55, 0x0a, 0x07, 0x08, 0x07, 0x10, + 0xf5, 0xe6, 0xd9, 0x55, 0x0a, 0x07, 0x08, 0x08, 0x10, 0xc1, 0xcc, 0xa7, + 0x41, 0x0a, 0x27, 0x18, 0x8f, 0x4e, 0x9a, 0x02, 0x21, 0x0a, 0x13, 0x08, + 0xf0, 0x1f, 0x10, 0x01, 0x18, 0x00, 0x20, 0x00, 0x28, 0x00, 0x30, 0x00, + 0x38, 0x00, 0x40, 0x00, 0x48, 0x00, 0x10, 0x01, 0x18, 0x07, 0x20, 0x06, + 0x28, 0x0b, 0x30, 0x01, 0x38, 0x01, 0x0a, 0xd5, 0x01, 0x18, 0x8f, 0x4e, + 0x8a, 0x02, 0xce, 0x01, 0x0a, 0x06, 0x08, 0x80, 0x80, 0x02, 0x20, 0x00, + 0x12, 0xa5, 0x01, 0x0a, 0xa2, 0x01, 0x0a, 0x0b, 0x61, 0x6e, 0x64, 0x72, + 0x6f, 0x69, 0x64, 0x2e, 0x6c, 0x6f, 0x67, 0x10, 0x00, 0x18, 0x00, 0x20, + 0x00, 0xa2, 0x06, 0x04, 0x50, 0x00, 0x58, 0x00, 0xaa, 0x06, 0x02, 0x0a, + 0x00, 0xb2, 0x06, 0x08, 0x08, 0x00, 0x10, 0x00, 0x18, 0x00, 0x20, 0x00, + 0xba, 0x06, 0x06, 0x10, 0x00, 0x18, 0x00, 0x20, 0x00, 0xc2, 0x06, 0x06, + 0x08, 0x00, 0x18, 0x00, 0x28, 0x00, 0xca, 0x06, 0x0a, 0x08, 0x00, 0x28, + 0x00, 0x32, 0x04, 0x28, 0x00, 0x30, 0x00, 0xd2, 0x06, 0x02, 0x08, 0x00, + 0xda, 0x06, 0x02, 0x18, 0x00, 0xc2, 0x3e, 0x00, 0xfa, 0xff, 0xff, 0xff, + 0x07, 0x46, 0x08, 0x00, 0x10, 0x00, 0x18, 0x00, 0x20, 0x00, 0x28, 0x00, + 0x32, 0x3a, 0x08, 0x00, 0x10, 0x00, 0x18, 0x00, 0x20, 0x00, 0x29, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x00, 0x45, 0x00, 0x00, + 0x00, 0x00, 0x49, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, + 0x00, 0x00, 0x00, 0x00, 0x58, 0x00, 0x60, 0x00, 0x6a, 0x00, 0x72, 0x00, + 0x18, 0xe0, 0xd4, 0x03, 0x20, 0x00, 0x28, 0x00, 0x3a, 0x06, 0x08, 0x00, + 0x10, 0x00, 0x18, 0x00, 0x40, 0x00, 0x48, 0x00, 0x50, 0x00, 0x5a, 0x02, + 0x08, 0x00, 0x60, 0x00, 0x68, 0x00, 0x0a, 0x94, 0x01, 0x40, 0xd9, 0xf4, + 0x98, 0x96, 0xbe, 0x54, 0xba, 0x02, 0x84, 0x81, 0x80, 0x00, 0x0a, 0xff, + 0x80, 0x80, 0x00, 0x38, 0x04, 0x32, 0x08, 0x70, 0x65, 0x72, 0x66, 0x65, + 0x74, 0x74, 0x6f, 0x42, 0x5d, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, + 0x5f, 0x6c, 0x6f, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x2e, 0x63, 0x63, 0x3a, 0x31, 0x35, 0x35, 0x20, + 0x53, 0x74, 0x61, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x20, 0x41, 0x6e, 0x64, + 0x72, 0x6f, 0x69, 0x64, 0x20, 0x6c, 0x6f, 0x67, 0x20, 0x64, 0x61, 0x74, + 0x61, 0x20, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x3a, 0x20, 0x73, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x20, 0x74, 0x61, 0x69, 0x6c, 0x3d, 0x31, 0x20, + 0x6c, 0x69, 0x64, 0x73, 0x3d, 0x30, 0x2c, 0x32, 0x2c, 0x33, 0x2c, 0x34, + 0x2c, 0x37, 0x28, 0xdb, 0xe6, 0x9b, 0xfb, 0xeb, 0xdb, 0xa5, 0xbc, 0x15, + 0x08, 0x00, 0x10, 0xb5, 0x58, 0x18, 0xb5, 0x58, 0x20, 0x00, 0x18, 0x8f, + 0x4e, 0x0a, 0xf4, 0x0a, 0x40, 0xb2, 0x84, 0xde, 0xdd, 0x8f, 0x55, 0xba, + 0x02, 0xe4, 0x8a, 0x80, 0x00, 0x0a, 0xe4, 0x80, 0x80, 0x00, 0x38, 0x04, + 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, + 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x38, 0x72, 0x6d, + 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x63, 0x62, 0x3a, 0x20, 0x63, 0x6c, + 0x6e, 0x74, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x31, 0x34, 0x20, 0x63, 0x6f, + 0x6e, 0x6e, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x37, 0x32, 0x31, 0x64, 0x61, + 0x30, 0x37, 0x31, 0x30, 0x30, 0x0a, 0x28, 0x92, 0xad, 0xf6, 0xf0, 0xbd, + 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, 0xe9, 0x06, + 0x20, 0x00, 0x0a, 0xf3, 0x80, 0x80, 0x00, 0x38, 0x06, 0x32, 0x0b, 0x72, + 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x4e, + 0x49, 0x4e, 0x46, 0x4f, 0x3a, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x5f, 0x72, 0x77, 0x5f, 0x69, 0x6f, 0x76, 0x65, + 0x63, 0x5f, 0x63, 0x62, 0x3a, 0x20, 0x57, 0x72, 0x69, 0x74, 0x65, 0x20, + 0x69, 0x6f, 0x76, 0x65, 0x63, 0x20, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x20, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x20, 0x66, + 0x6f, 0x72, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, + 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x63, 0x28, 0x9e, 0xa1, 0xea, 0xf0, 0xbd, + 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x07, 0x10, 0xe9, 0x06, 0x18, 0xe9, 0x06, + 0x20, 0x00, 0x0a, 0xfd, 0x80, 0x80, 0x00, 0x38, 0x06, 0x32, 0x0b, 0x72, + 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x58, + 0x49, 0x4e, 0x46, 0x4f, 0x3a, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, + 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, 0x20, 0x43, 0x61, 0x6c, 0x6c, + 0x69, 0x6e, 0x67, 0x20, 0x57, 0x72, 0x69, 0x74, 0x65, 0x20, 0x5b, 0x6f, + 0x66, 0x66, 0x73, 0x65, 0x74, 0x3d, 0x30, 0x2c, 0x20, 0x73, 0x69, 0x7a, + 0x65, 0x3d, 0x36, 0x35, 0x35, 0x33, 0x36, 0x5d, 0x66, 0x6f, 0x72, 0x20, + 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, + 0x66, 0x73, 0x63, 0x21, 0x28, 0xe6, 0xc4, 0x80, 0xf1, 0xbd, 0xdc, 0xa5, + 0xbc, 0x15, 0x08, 0x07, 0x10, 0xf6, 0x0b, 0x18, 0xf6, 0x0b, 0x20, 0x00, + 0x0a, 0x80, 0x81, 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, 0x65, 0x6e, + 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x42, 0x54, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x5f, 0x72, 0x77, 0x5f, 0x69, 0x6f, 0x76, 0x65, + 0x63, 0x5f, 0x63, 0x62, 0x3a, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, + 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x63, 0x3a, 0x20, 0x72, + 0x65, 0x71, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x31, 0x34, 0x20, 0x6d, 0x73, + 0x67, 0x5f, 0x69, 0x64, 0x3d, 0x33, 0x3a, 0x20, 0x52, 0x2f, 0x57, 0x20, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x20, 0x72, 0x65, 0x63, 0x65, + 0x69, 0x76, 0x65, 0x64, 0x0a, 0x28, 0xc3, 0xe0, 0xf9, 0xf0, 0xbd, 0xdc, + 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, 0xe9, 0x06, 0x20, + 0x00, 0x0a, 0xc1, 0x80, 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, 0x65, + 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x42, 0x15, 0x77, 0x61, 0x6b, 0x65, 0x6c, 0x6f, + 0x63, 0x6b, 0x20, 0x61, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x3a, + 0x20, 0x31, 0x0a, 0x28, 0xb4, 0xc7, 0x8a, 0xf1, 0xbd, 0xdc, 0xa5, 0xbc, + 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, 0xe9, 0x06, 0x20, 0x00, 0x0a, + 0x84, 0x81, 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, + 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x42, 0x58, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, + 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, + 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x63, 0x3a, 0x20, + 0x55, 0x6e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x20, 0x77, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x20, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x20, 0x28, 0x74, + 0x68, 0x5f, 0x69, 0x64, 0x3a, 0x20, 0x34, 0x39, 0x30, 0x31, 0x31, 0x34, + 0x39, 0x35, 0x34, 0x34, 0x36, 0x34, 0x29, 0x0a, 0x28, 0x8e, 0xe0, 0x8e, + 0xf1, 0xbd, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, + 0xf6, 0x0b, 0x20, 0x00, 0x0a, 0x83, 0x81, 0x80, 0x00, 0x38, 0x04, 0x32, + 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, + 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x57, 0x72, 0x6d, 0x74, + 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, 0x20, + 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, + 0x66, 0x73, 0x63, 0x3a, 0x20, 0x72, 0x65, 0x71, 0x5f, 0x68, 0x3d, 0x30, + 0x78, 0x31, 0x34, 0x20, 0x6d, 0x73, 0x67, 0x5f, 0x69, 0x64, 0x3d, 0x33, + 0x3a, 0x20, 0x42, 0x79, 0x74, 0x65, 0x73, 0x20, 0x77, 0x72, 0x69, 0x74, + 0x74, 0x65, 0x6e, 0x20, 0x3d, 0x20, 0x36, 0x35, 0x35, 0x33, 0x36, 0x0a, + 0x28, 0x8a, 0xe1, 0xa0, 0xf2, 0xbd, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, + 0x10, 0xe9, 0x06, 0x18, 0xf6, 0x0b, 0x20, 0x00, 0x0a, 0x88, 0x81, 0x80, + 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, + 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, + 0x5c, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, + 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, + 0x61, 0x64, 0x3a, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, + 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x63, 0x3a, 0x20, 0x72, 0x65, 0x71, + 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x31, 0x34, 0x20, 0x6d, 0x73, 0x67, 0x5f, + 0x69, 0x64, 0x3d, 0x33, 0x3a, 0x20, 0x53, 0x65, 0x6e, 0x64, 0x20, 0x72, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x3a, 0x20, 0x72, 0x65, 0x73, + 0x3d, 0x30, 0x20, 0x65, 0x72, 0x72, 0x3d, 0x30, 0x0a, 0x28, 0xf8, 0x89, + 0xa2, 0xf2, 0xbd, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, + 0x18, 0xf6, 0x0b, 0x20, 0x00, 0x0a, 0xae, 0x81, 0x80, 0x00, 0x38, 0x04, + 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, + 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x81, 0x01, 0x72, + 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, + 0x3a, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, + 0x6d, 0x5f, 0x66, 0x73, 0x63, 0x3a, 0x20, 0x41, 0x62, 0x6f, 0x75, 0x74, + 0x20, 0x74, 0x6f, 0x20, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x20, 0x72, 0x6d, + 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x20, 0x63, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x20, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x20, + 0x28, 0x74, 0x68, 0x5f, 0x69, 0x64, 0x3a, 0x20, 0x34, 0x39, 0x30, 0x31, + 0x31, 0x34, 0x39, 0x35, 0x34, 0x34, 0x36, 0x34, 0x29, 0x20, 0x77, 0x61, + 0x6b, 0x65, 0x6c, 0x6f, 0x63, 0x6b, 0x20, 0x72, 0x65, 0x6c, 0x65, 0x61, + 0x73, 0x65, 0x64, 0x3a, 0x20, 0x31, 0x0a, 0x0a, 0x28, 0xb9, 0xcb, 0xa9, + 0xf2, 0xbd, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, + 0xf6, 0x0b, 0x20, 0x00, 0x0a, 0xf4, 0x80, 0x80, 0x00, 0x38, 0x06, 0x32, + 0x0b, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, + 0x42, 0x4f, 0x49, 0x4e, 0x46, 0x4f, 0x3a, 0x72, 0x6d, 0x74, 0x5f, 0x73, + 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, 0x20, 0x44, 0x6f, + 0x6e, 0x65, 0x20, 0x57, 0x72, 0x69, 0x74, 0x65, 0x20, 0x28, 0x62, 0x79, + 0x74, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x36, 0x35, 0x35, 0x33, 0x36, 0x29, + 0x20, 0x66, 0x6f, 0x72, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, + 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x63, 0x21, 0x28, 0xd6, 0xbd, + 0x8f, 0xf2, 0xbd, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x07, 0x10, 0xf6, 0x0b, + 0x18, 0xf6, 0x0b, 0x20, 0x00, 0x0a, 0xe7, 0x80, 0x80, 0x00, 0x38, 0x04, + 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, + 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x3b, 0x72, 0x6d, + 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x64, 0x69, + 0x73, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x63, 0x62, 0x3a, + 0x20, 0x63, 0x6c, 0x6e, 0x74, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x31, 0x34, + 0x20, 0x63, 0x6f, 0x6e, 0x6e, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x37, 0x32, + 0x31, 0x64, 0x61, 0x30, 0x37, 0x31, 0x30, 0x30, 0x0a, 0x28, 0x92, 0xd5, + 0xc7, 0xf2, 0xbd, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, + 0x18, 0xe9, 0x06, 0x20, 0x00, 0x18, 0x8f, 0x4e, 0x0a, 0x70, 0x40, 0x92, + 0xaf, 0xd5, 0x8d, 0x90, 0x55, 0xba, 0x02, 0xe0, 0x80, 0x80, 0x00, 0x0a, + 0xdb, 0x80, 0x80, 0x00, 0x38, 0x04, 0x32, 0x08, 0x70, 0x65, 0x72, 0x66, + 0x65, 0x74, 0x74, 0x6f, 0x42, 0x39, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, + 0x64, 0x5f, 0x6c, 0x6f, 0x67, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x63, 0x63, 0x3a, 0x32, 0x39, 0x31, + 0x20, 0x53, 0x65, 0x65, 0x6e, 0x20, 0x31, 0x31, 0x20, 0x41, 0x6e, 0x64, + 0x72, 0x6f, 0x69, 0x64, 0x20, 0x6c, 0x6f, 0x67, 0x20, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x73, 0x28, 0xd0, 0x8f, 0xfa, 0xf4, 0xbd, 0xdc, 0xa5, 0xbc, + 0x15, 0x08, 0x00, 0x10, 0xb5, 0x58, 0x18, 0xb5, 0x58, 0x20, 0x00, 0x18, + 0x8f, 0x4e, 0x0a, 0xfa, 0x0a, 0x40, 0x9c, 0xc7, 0xf8, 0xbc, 0x90, 0x55, + 0xba, 0x02, 0xea, 0x8a, 0x80, 0x00, 0x0a, 0xe4, 0x80, 0x80, 0x00, 0x38, + 0x04, 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, + 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x38, 0x72, + 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x63, 0x62, 0x3a, 0x20, 0x63, + 0x6c, 0x6e, 0x74, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x31, 0x35, 0x20, 0x63, + 0x6f, 0x6e, 0x6e, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x37, 0x32, 0x31, 0x64, + 0x61, 0x30, 0x37, 0x31, 0x30, 0x30, 0x0a, 0x28, 0xb4, 0xa2, 0x8b, 0xa5, + 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, 0xe9, + 0x06, 0x20, 0x00, 0x0a, 0x80, 0x81, 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, + 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, + 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x54, 0x72, 0x6d, 0x74, 0x5f, + 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x72, 0x77, 0x5f, 0x69, + 0x6f, 0x76, 0x65, 0x63, 0x5f, 0x63, 0x62, 0x3a, 0x20, 0x2f, 0x62, 0x6f, + 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x31, + 0x3a, 0x20, 0x72, 0x65, 0x71, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x31, 0x35, + 0x20, 0x6d, 0x73, 0x67, 0x5f, 0x69, 0x64, 0x3d, 0x33, 0x3a, 0x20, 0x52, + 0x2f, 0x57, 0x20, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x20, 0x72, + 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x0a, 0x28, 0x87, 0xf1, 0x8e, + 0xa5, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, + 0xe9, 0x06, 0x20, 0x00, 0x0a, 0xc1, 0x80, 0x80, 0x00, 0x38, 0x04, 0x32, + 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, + 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x15, 0x77, 0x61, 0x6b, + 0x65, 0x6c, 0x6f, 0x63, 0x6b, 0x20, 0x61, 0x63, 0x71, 0x75, 0x69, 0x72, + 0x65, 0x64, 0x3a, 0x20, 0x31, 0x0a, 0x28, 0x8e, 0x8e, 0x9e, 0xa5, 0xbe, + 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, 0xe9, 0x06, + 0x20, 0x00, 0x0a, 0x84, 0x81, 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, + 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x58, 0x72, 0x6d, 0x74, 0x5f, 0x73, + 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, 0x20, 0x2f, 0x62, + 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, + 0x31, 0x3a, 0x20, 0x55, 0x6e, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x20, 0x77, + 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x20, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, + 0x20, 0x28, 0x74, 0x68, 0x5f, 0x69, 0x64, 0x3a, 0x20, 0x34, 0x39, 0x30, + 0x31, 0x32, 0x38, 0x37, 0x30, 0x30, 0x36, 0x34, 0x30, 0x29, 0x0a, 0x28, + 0x9a, 0xa4, 0xa1, 0xa5, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, + 0xe9, 0x06, 0x18, 0xf1, 0x0b, 0x20, 0x00, 0x0a, 0xf3, 0x80, 0x80, 0x00, + 0x38, 0x06, 0x32, 0x0b, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x42, 0x4e, 0x49, 0x4e, 0x46, 0x4f, 0x3a, 0x72, 0x6d, + 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x72, 0x77, + 0x5f, 0x69, 0x6f, 0x76, 0x65, 0x63, 0x5f, 0x63, 0x62, 0x3a, 0x20, 0x57, + 0x72, 0x69, 0x74, 0x65, 0x20, 0x69, 0x6f, 0x76, 0x65, 0x63, 0x20, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x20, 0x72, 0x65, 0x63, 0x65, 0x69, + 0x76, 0x65, 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x2f, 0x62, 0x6f, 0x6f, + 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x31, 0x28, + 0x8e, 0xb1, 0xff, 0xa4, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x07, 0x10, + 0xe9, 0x06, 0x18, 0xe9, 0x06, 0x20, 0x00, 0x0a, 0xff, 0x80, 0x80, 0x00, + 0x38, 0x06, 0x32, 0x0b, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x42, 0x5a, 0x49, 0x4e, 0x46, 0x4f, 0x3a, 0x72, 0x6d, + 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, + 0x20, 0x43, 0x61, 0x6c, 0x6c, 0x69, 0x6e, 0x67, 0x20, 0x57, 0x72, 0x69, + 0x74, 0x65, 0x20, 0x5b, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x3d, 0x30, + 0x2c, 0x20, 0x73, 0x69, 0x7a, 0x65, 0x3d, 0x32, 0x30, 0x39, 0x37, 0x31, + 0x35, 0x32, 0x5d, 0x66, 0x6f, 0x72, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, + 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x31, 0x21, 0x28, + 0xe6, 0xae, 0x92, 0xa5, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x07, 0x10, + 0xf1, 0x0b, 0x18, 0xf1, 0x0b, 0x20, 0x00, 0x0a, 0x85, 0x81, 0x80, 0x00, + 0x38, 0x04, 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, + 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x59, + 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, + 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, + 0x64, 0x3a, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, + 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x31, 0x3a, 0x20, 0x72, 0x65, 0x71, 0x5f, + 0x68, 0x3d, 0x30, 0x78, 0x31, 0x35, 0x20, 0x6d, 0x73, 0x67, 0x5f, 0x69, + 0x64, 0x3d, 0x33, 0x3a, 0x20, 0x42, 0x79, 0x74, 0x65, 0x73, 0x20, 0x77, + 0x72, 0x69, 0x74, 0x74, 0x65, 0x6e, 0x20, 0x3d, 0x20, 0x32, 0x30, 0x39, + 0x37, 0x31, 0x35, 0x32, 0x0a, 0x28, 0x96, 0x87, 0xd7, 0xb3, 0xbe, 0xdc, + 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, 0xf1, 0x0b, 0x20, + 0x00, 0x0a, 0x88, 0x81, 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, 0x65, + 0x6e, 0x64, 0x6f, 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x42, 0x5c, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, 0x20, 0x2f, 0x62, 0x6f, + 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x31, + 0x3a, 0x20, 0x72, 0x65, 0x71, 0x5f, 0x68, 0x3d, 0x30, 0x78, 0x31, 0x35, + 0x20, 0x6d, 0x73, 0x67, 0x5f, 0x69, 0x64, 0x3d, 0x33, 0x3a, 0x20, 0x53, + 0x65, 0x6e, 0x64, 0x20, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x3a, 0x20, 0x72, 0x65, 0x73, 0x3d, 0x30, 0x20, 0x65, 0x72, 0x72, 0x3d, + 0x30, 0x0a, 0x28, 0xda, 0xb3, 0xd8, 0xb3, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, + 0x08, 0x00, 0x10, 0xe9, 0x06, 0x18, 0xf1, 0x0b, 0x20, 0x00, 0x0a, 0xae, + 0x81, 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, + 0x72, 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x42, 0x81, 0x01, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, + 0x68, 0x72, 0x65, 0x61, 0x64, 0x3a, 0x20, 0x2f, 0x62, 0x6f, 0x6f, 0x74, + 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, 0x66, 0x73, 0x31, 0x3a, 0x20, + 0x41, 0x62, 0x6f, 0x75, 0x74, 0x20, 0x74, 0x6f, 0x20, 0x62, 0x6c, 0x6f, + 0x63, 0x6b, 0x20, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x20, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x20, 0x74, 0x68, + 0x72, 0x65, 0x61, 0x64, 0x20, 0x28, 0x74, 0x68, 0x5f, 0x69, 0x64, 0x3a, + 0x20, 0x34, 0x39, 0x30, 0x31, 0x32, 0x38, 0x37, 0x30, 0x30, 0x36, 0x34, + 0x30, 0x29, 0x20, 0x77, 0x61, 0x6b, 0x65, 0x6c, 0x6f, 0x63, 0x6b, 0x20, + 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x64, 0x3a, 0x20, 0x31, 0x0a, + 0x0a, 0x28, 0xe4, 0xa2, 0xe0, 0xb3, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08, + 0x00, 0x10, 0xe9, 0x06, 0x18, 0xf1, 0x0b, 0x20, 0x00, 0x0a, 0xe7, 0x80, + 0x80, 0x00, 0x38, 0x04, 0x32, 0x12, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, + 0x2e, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, + 0x42, 0x3b, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x5f, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x5f, 0x63, 0x62, 0x3a, 0x20, 0x63, 0x6c, 0x6e, 0x74, 0x5f, 0x68, 0x3d, + 0x30, 0x78, 0x31, 0x35, 0x20, 0x63, 0x6f, 0x6e, 0x6e, 0x5f, 0x68, 0x3d, + 0x30, 0x78, 0x37, 0x32, 0x31, 0x64, 0x61, 0x30, 0x37, 0x31, 0x30, 0x30, + 0x0a, 0x28, 0xeb, 0xea, 0x8f, 0xb4, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08, + 0x00, 0x10, 0xe9, 0x06, 0x18, 0xe9, 0x06, 0x20, 0x00, 0x0a, 0xf6, 0x80, + 0x80, 0x00, 0x38, 0x06, 0x32, 0x0b, 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x51, 0x49, 0x4e, 0x46, 0x4f, 0x3a, + 0x72, 0x6d, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, + 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, + 0x64, 0x3a, 0x20, 0x44, 0x6f, 0x6e, 0x65, 0x20, 0x57, 0x72, 0x69, 0x74, + 0x65, 0x20, 0x28, 0x62, 0x79, 0x74, 0x65, 0x73, 0x20, 0x3d, 0x20, 0x32, + 0x30, 0x39, 0x37, 0x31, 0x35, 0x32, 0x29, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6d, 0x5f, + 0x66, 0x73, 0x31, 0x21, 0x28, 0x9e, 0xa9, 0xc5, 0xb3, 0xbe, 0xdc, 0xa5, + 0xbc, 0x15, 0x08, 0x07, 0x10, 0xf1, 0x0b, 0x18, 0xf1, 0x0b, 0x20, 0x00, + 0x18, 0x8f, 0x4e, 0x0a, 0x70, 0x40, 0xd2, 0x9f, 0x8f, 0xed, 0x90, 0x55, + 0xba, 0x02, 0xe0, 0x80, 0x80, 0x00, 0x0a, 0xdb, 0x80, 0x80, 0x00, 0x38, + 0x04, 0x32, 0x08, 0x70, 0x65, 0x72, 0x66, 0x65, 0x74, 0x74, 0x6f, 0x42, + 0x39, 0x61, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x5f, 0x6c, 0x6f, 0x67, + 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x2e, 0x63, 0x63, 0x3a, 0x32, 0x39, 0x31, 0x20, 0x53, 0x65, 0x65, 0x6e, + 0x20, 0x31, 0x31, 0x20, 0x41, 0x6e, 0x64, 0x72, 0x6f, 0x69, 0x64, 0x20, + 0x6c, 0x6f, 0x67, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x28, 0xf4, + 0x9d, 0x96, 0xd4, 0xbe, 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xb5, + 0x58, 0x18, 0xb5, 0x58, 0x20, 0x00, 0x18, 0x8f, 0x4e, 0x0a, 0x4e, 0x40, + 0xfe, 0xf3, 0x83, 0xd1, 0x9e, 0x55, 0xba, 0x02, 0xbe, 0x80, 0x80, 0x00, + 0x0a, 0xb9, 0x80, 0x80, 0x00, 0x38, 0x06, 0x32, 0x00, 0x42, 0x1f, 0x74, + 0x75, 0x69, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x3a, 0x73, 0x75, 0x69, 0x73, + 0x76, 0x63, 0x20, 0x20, 0x65, 0x78, 0x69, 0x73, 0x20, 0x61, 0x6c, 0x72, + 0x65, 0x61, 0x64, 0x79, 0x20, 0x30, 0x28, 0x9f, 0x92, 0xdf, 0xdf, 0xcc, + 0xdc, 0xa5, 0xbc, 0x15, 0x08, 0x00, 0x10, 0xd9, 0x06, 0x18, 0xd9, 0x06, + 0x20, 0x00, 0x18, 0x8f, 0x4e, 0x0a, 0x1b, 0x40, 0x9b, 0x83, 0xa7, 0xba, + 0xba, 0x55, 0xba, 0x02, 0x8b, 0x80, 0x80, 0x00, 0x12, 0x86, 0x80, 0x80, + 0x00, 0x08, 0x1a, 0x18, 0x00, 0x10, 0x00, 0x18, 0x8f, 0x4e}; + + static const uint8_t kFilterBytecode[]{ + 0x0b, 0x01, 0x00, 0x0b, 0x60, 0x13, 0x02, 0x19, 0x23, 0x08, 0x2b, 0x09, + 0x33, 0x0f, 0x3b, 0x10, 0x41, 0x4b, 0x05, 0x51, 0x5b, 0x12, 0x63, 0x72, + 0x69, 0x8b, 0x02, 0x21, 0x93, 0x02, 0x33, 0x9b, 0x02, 0x35, 0xa1, 0x02, + 0xab, 0x02, 0x37, 0xb3, 0x02, 0x07, 0xbb, 0x02, 0x3e, 0xc3, 0x02, 0x3d, + 0xca, 0x02, 0x02, 0xdb, 0x02, 0x1b, 0xe3, 0x02, 0x11, 0xeb, 0x02, 0x40, + 0xf3, 0x02, 0x04, 0xfb, 0x02, 0x41, 0x83, 0x03, 0x34, 0x8b, 0x03, 0x42, + 0x91, 0x03, 0x9b, 0x03, 0x43, 0xa3, 0x03, 0x46, 0xab, 0x03, 0x49, 0xb3, + 0x03, 0x04, 0xbb, 0x03, 0x07, 0xc3, 0x03, 0x4b, 0xcb, 0x03, 0x4c, 0xd1, + 0x03, 0xdb, 0x03, 0x73, 0xe3, 0x03, 0x5f, 0xeb, 0x03, 0x5b, 0xf3, 0x03, + 0x4d, 0xfb, 0x03, 0x04, 0x83, 0x04, 0x5d, 0x8b, 0x04, 0x4e, 0x93, 0x04, + 0x4f, 0x9b, 0x04, 0x50, 0xa3, 0x04, 0x51, 0xab, 0x04, 0x3c, 0xb3, 0x04, + 0x0e, 0xbb, 0x04, 0x04, 0xc3, 0x04, 0x69, 0xcb, 0x04, 0x53, 0xd3, 0x04, + 0x3c, 0xdb, 0x04, 0x04, 0xe3, 0x04, 0x56, 0xeb, 0x04, 0x58, 0xf3, 0x04, + 0x5a, 0xa3, 0x38, 0x70, 0x00, 0x0b, 0x03, 0x13, 0x04, 0x19, 0x00, 0x0a, + 0x03, 0x23, 0x04, 0x29, 0x00, 0x0a, 0x03, 0x00, 0x0b, 0x06, 0x11, 0x00, + 0x0a, 0x0a, 0x5b, 0x07, 0x62, 0x03, 0x00, 0x0a, 0x04, 0x00, 0x0a, 0x02, + 0x1b, 0x04, 0x00, 0x0b, 0x0a, 0x13, 0x0d, 0x1b, 0x0e, 0x21, 0x2b, 0x0e, + 0x00, 0x0a, 0x0d, 0x73, 0x0b, 0x7a, 0x02, 0x00, 0x0a, 0x09, 0x53, 0x0c, + 0x00, 0x0a, 0x02, 0x1b, 0x0c, 0x23, 0x0c, 0x2a, 0x04, 0x00, 0x0a, 0x05, + 0x00, 0x0a, 0x02, 0x00, 0x0b, 0x07, 0x11, 0x00, 0x0b, 0x0e, 0x13, 0x0e, + 0x1b, 0x11, 0x22, 0x02, 0x33, 0x0e, 0x39, 0x43, 0x0e, 0x49, 0x53, 0x0e, + 0x00, 0x0a, 0x08, 0x00, 0x0a, 0x03, 0x23, 0x13, 0x2b, 0x15, 0x33, 0x20, + 0x42, 0x05, 0x82, 0x01, 0x02, 0xa1, 0x01, 0xab, 0x01, 0x0e, 0xb2, 0x01, + 0x02, 0xc3, 0x01, 0x16, 0xcb, 0x01, 0x0e, 0xd3, 0x01, 0x15, 0xdb, 0x01, + 0x0e, 0xe3, 0x01, 0x07, 0xeb, 0x01, 0x1e, 0xf2, 0x01, 0x02, 0x83, 0x02, + 0x1f, 0x8b, 0x02, 0x07, 0x91, 0x02, 0x9b, 0x02, 0x0e, 0xa1, 0x02, 0xb3, + 0x02, 0x04, 0xbb, 0x02, 0x15, 0xc3, 0x02, 0x15, 0xcb, 0x02, 0x04, 0xd1, + 0x02, 0xdb, 0x02, 0x15, 0xe2, 0x02, 0x03, 0x00, 0x0a, 0x07, 0x43, 0x14, + 0x4a, 0x02, 0x5b, 0x13, 0x63, 0x13, 0x00, 0x0a, 0x02, 0x1b, 0x14, 0x23, + 0x14, 0x2a, 0x04, 0x00, 0x09, 0x00, 0x0b, 0x17, 0x12, 0x0c, 0x73, 0x19, + 0x7b, 0x1c, 0x83, 0x01, 0x1d, 0x8b, 0x01, 0x1b, 0x00, 0x0b, 0x0d, 0x13, + 0x18, 0x00, 0x0a, 0x2e, 0x00, 0x0a, 0x03, 0x23, 0x1a, 0x2b, 0x1a, 0x33, + 0x1b, 0x00, 0x0a, 0x09, 0x53, 0x07, 0x00, 0x0a, 0x07, 0x00, 0x09, 0x13, + 0x1a, 0x00, 0x0a, 0x03, 0x23, 0x1a, 0x00, 0x0a, 0x03, 0x23, 0x0e, 0x2a, + 0x02, 0x00, 0x0a, 0x0a, 0x00, 0x0a, 0x04, 0x32, 0x0a, 0x92, 0x01, 0x02, + 0x00, 0x0b, 0x22, 0x13, 0x23, 0x1a, 0x03, 0x33, 0x04, 0x3b, 0x07, 0x42, + 0x03, 0x5b, 0x15, 0x62, 0x03, 0x81, 0x01, 0x8b, 0x01, 0x32, 0x92, 0x01, + 0x02, 0xa3, 0x01, 0x1b, 0xab, 0x01, 0x15, 0xb2, 0x01, 0x03, 0xcb, 0x01, + 0x0d, 0xda, 0x01, 0x05, 0x83, 0x02, 0x15, 0x00, 0x09, 0x21, 0x00, 0x0b, + 0x24, 0x12, 0x02, 0x00, 0x0a, 0x04, 0x32, 0x03, 0xa3, 0x06, 0x25, 0xab, + 0x06, 0x0d, 0xb3, 0x06, 0x26, 0xbb, 0x06, 0x27, 0xc3, 0x06, 0x1b, 0xcb, + 0x06, 0x28, 0xd3, 0x06, 0x07, 0xdb, 0x06, 0x2b, 0xe3, 0x06, 0x07, 0xeb, + 0x06, 0x15, 0xf3, 0x06, 0x2a, 0xfb, 0x06, 0x2c, 0x83, 0x07, 0x0e, 0x8b, + 0x07, 0x07, 0x93, 0x07, 0x15, 0x9b, 0x07, 0x2f, 0xc1, 0x3e, 0xcb, 0x3e, + 0x30, 0xf9, 0xff, 0xff, 0xff, 0x07, 0x00, 0x0a, 0x03, 0x52, 0x02, 0x63, + 0x15, 0x6a, 0x02, 0x00, 0x0a, 0x05, 0x33, 0x0e, 0x00, 0x0a, 0x04, 0x32, + 0x03, 0x00, 0x0a, 0x02, 0x22, 0x02, 0x33, 0x29, 0x3a, 0x05, 0x6a, 0x0e, + 0x00, 0x2a, 0x02, 0x00, 0x0a, 0x02, 0x1b, 0x0e, 0x22, 0x04, 0x00, 0x09, + 0x1a, 0x02, 0x00, 0x0a, 0x0d, 0x7b, 0x2d, 0x83, 0x01, 0x2e, 0x8a, 0x01, + 0x02, 0x00, 0x0a, 0x02, 0x1b, 0x0e, 0x21, 0x00, 0x0b, 0x0d, 0x11, 0x00, + 0x09, 0xa3, 0x06, 0x0e, 0x00, 0x0a, 0x05, 0x33, 0x31, 0x00, 0x0a, 0x0e, + 0x00, 0x09, 0x13, 0x0d, 0x19, 0x00, 0x09, 0x13, 0x34, 0x1a, 0x02, 0x00, + 0x0a, 0x09, 0x00, 0x0b, 0x36, 0x12, 0x09, 0x5b, 0x0d, 0x00, 0x0a, 0x13, + 0x00, 0x0b, 0x0e, 0x13, 0x07, 0x1b, 0x0e, 0x23, 0x11, 0x2b, 0x38, 0x32, + 0x02, 0x00, 0x09, 0x13, 0x3b, 0x1a, 0x02, 0x2b, 0x39, 0x32, 0x09, 0x00, + 0x0a, 0x03, 0x23, 0x3a, 0x2a, 0x02, 0x00, 0x0b, 0x04, 0x00, 0x0a, 0x06, + 0x42, 0x02, 0x00, 0x0a, 0x06, 0x00, 0x0b, 0x07, 0x13, 0x04, 0x00, 0x0b, + 0x3f, 0x13, 0x04, 0x00, 0x0a, 0x08, 0x4b, 0x07, 0x00, 0x0b, 0x07, 0x12, + 0x02, 0x00, 0x0b, 0x0d, 0x12, 0x02, 0x00, 0x0a, 0x06, 0x3b, 0x0e, 0x42, + 0x02, 0x00, 0x0b, 0x44, 0x12, 0x02, 0x00, 0x0b, 0x45, 0x13, 0x45, 0x00, + 0x09, 0x13, 0x04, 0x1b, 0x0e, 0x00, 0x0b, 0x47, 0x13, 0x04, 0x19, 0x00, + 0x0b, 0x48, 0x13, 0x0d, 0x1a, 0x03, 0x00, 0x0a, 0x03, 0x2a, 0x06, 0x00, + 0x0a, 0x05, 0x33, 0x0e, 0x3b, 0x4a, 0x42, 0x08, 0x00, 0x0b, 0x0e, 0x13, + 0x0e, 0x1b, 0x0e, 0x00, 0x09, 0x13, 0x1b, 0x23, 0x0e, 0x2a, 0x02, 0x3b, + 0x0e, 0x43, 0x0e, 0x4b, 0x11, 0x00, 0x0b, 0x0d, 0x00, 0x0a, 0x08, 0x4b, + 0x07, 0x82, 0x01, 0x05, 0x00, 0x0b, 0x0d, 0x13, 0x3c, 0x00, 0x0a, 0x06, + 0x82, 0x01, 0x03, 0x9b, 0x01, 0x15, 0x00, 0x0b, 0x0e, 0x00, 0x09, 0x13, + 0x52, 0x00, 0x0a, 0x0f, 0x00, 0x0a, 0x02, 0x1b, 0x54, 0x00, 0x09, 0x13, + 0x55, 0x1b, 0x07, 0x00, 0x0a, 0x04, 0x2b, 0x07, 0x00, 0x0b, 0x04, 0x13, + 0x11, 0x1b, 0x0d, 0x23, 0x57, 0x2b, 0x15, 0x00, 0x0a, 0x0b, 0x00, 0x0b, + 0x59, 0x12, 0x02, 0x23, 0x0e, 0x00, 0x0b, 0x07, 0x00, 0x0a, 0x02, 0x1b, + 0x0e, 0x00, 0x0a, 0x02, 0x1b, 0x5c, 0x00, 0x09, 0x13, 0x04, 0x00, 0x0a, + 0x02, 0x1b, 0x5e, 0x00, 0x0a, 0x02, 0x1b, 0x0e, 0x23, 0x0e, 0x00, 0x0a, + 0x02, 0x1b, 0x1b, 0x23, 0x11, 0x29, 0x33, 0x0d, 0x3b, 0x0e, 0x43, 0x3c, + 0x00, 0x09, 0x13, 0x61, 0x19, 0x23, 0x57, 0x00, 0x0a, 0x02, 0x1b, 0x0e, + 0x23, 0x1b, 0x5b, 0x0e, 0x63, 0x04, 0x6b, 0x0e, 0x73, 0x04, 0x7b, 0x04, + 0x83, 0x01, 0x04, 0x8b, 0x01, 0x0d, 0x93, 0x01, 0x04, 0x9b, 0x01, 0x04, + 0xa3, 0x01, 0x0d, 0xab, 0x01, 0x15, 0xb3, 0x01, 0x15, 0xbb, 0x01, 0x0e, + 0xc3, 0x01, 0x15, 0xcb, 0x01, 0x15, 0xd3, 0x01, 0x15, 0xdb, 0x01, 0x0d, + 0xe3, 0x01, 0x3c, 0xeb, 0x01, 0x04, 0xf3, 0x01, 0x3c, 0xfb, 0x01, 0x0d, + 0x83, 0x02, 0x3c, 0x8b, 0x02, 0x1b, 0x93, 0x02, 0x3c, 0x9b, 0x02, 0x0d, + 0xa3, 0x02, 0x04, 0xab, 0x02, 0x0e, 0xb3, 0x02, 0x0e, 0xbb, 0x02, 0x0e, + 0xc3, 0x02, 0x04, 0xcb, 0x02, 0x0d, 0xd3, 0x02, 0x0d, 0xdb, 0x02, 0x07, + 0xe3, 0x02, 0x04, 0xeb, 0x02, 0x1b, 0xf3, 0x02, 0x04, 0xfb, 0x02, 0x15, + 0x83, 0x03, 0x0e, 0x8b, 0x03, 0x15, 0x93, 0x03, 0x1b, 0x9b, 0x03, 0x15, + 0xa3, 0x03, 0x0d, 0xab, 0x03, 0x15, 0xb3, 0x03, 0x15, 0xbb, 0x03, 0x15, + 0xc3, 0x03, 0x15, 0xcb, 0x03, 0x15, 0xd3, 0x03, 0x0e, 0xdb, 0x03, 0x0d, + 0xe3, 0x03, 0x15, 0xeb, 0x03, 0x15, 0xf3, 0x03, 0x15, 0xfb, 0x03, 0x15, + 0x83, 0x04, 0x15, 0x8b, 0x04, 0x04, 0x93, 0x04, 0x0e, 0x9b, 0x04, 0x0d, + 0xa3, 0x04, 0x04, 0xab, 0x04, 0x04, 0xb3, 0x04, 0x04, 0xbb, 0x04, 0x0d, + 0xc3, 0x04, 0x04, 0xcb, 0x04, 0x04, 0xd3, 0x04, 0x04, 0xdb, 0x04, 0x04, + 0xe3, 0x04, 0x0e, 0xeb, 0x04, 0x07, 0xf3, 0x04, 0x07, 0xfb, 0x04, 0x62, + 0x83, 0x05, 0x04, 0x8b, 0x05, 0x07, 0x93, 0x05, 0x11, 0x9b, 0x05, 0x0d, + 0xa3, 0x05, 0x62, 0xab, 0x05, 0x0e, 0xb3, 0x05, 0x04, 0xbb, 0x05, 0x1b, + 0xc3, 0x05, 0x04, 0xcb, 0x05, 0x15, 0xd3, 0x05, 0x15, 0xdb, 0x05, 0x11, + 0xe3, 0x05, 0x0e, 0xeb, 0x05, 0x0e, 0xf3, 0x05, 0x1f, 0xfb, 0x05, 0x04, + 0x83, 0x06, 0x15, 0x8b, 0x06, 0x0d, 0x93, 0x06, 0x0d, 0x9b, 0x06, 0x0d, + 0xa3, 0x06, 0x3c, 0xab, 0x06, 0x3c, 0xb3, 0x06, 0x3c, 0xbb, 0x06, 0x3c, + 0xc3, 0x06, 0x07, 0xcb, 0x06, 0x07, 0xd3, 0x06, 0x07, 0xdb, 0x06, 0x15, + 0xe3, 0x06, 0x04, 0xeb, 0x06, 0x0e, 0xf3, 0x06, 0x07, 0xfb, 0x06, 0x04, + 0x83, 0x07, 0x04, 0x8b, 0x07, 0x04, 0x93, 0x07, 0x0d, 0x9b, 0x07, 0x0d, + 0xa3, 0x07, 0x0d, 0xab, 0x07, 0x0d, 0xb3, 0x07, 0x0d, 0xbb, 0x07, 0x0d, + 0xc3, 0x07, 0x3c, 0xcb, 0x07, 0x04, 0xd3, 0x07, 0x0d, 0xdb, 0x07, 0x15, + 0xe3, 0x07, 0x3c, 0xeb, 0x07, 0x3c, 0xf3, 0x07, 0x1b, 0x83, 0x08, 0x1b, + 0x8b, 0x08, 0x3c, 0x93, 0x08, 0x0d, 0x9b, 0x08, 0x0d, 0xa3, 0x08, 0x04, + 0xab, 0x08, 0x0e, 0xb3, 0x08, 0x07, 0xbb, 0x08, 0x57, 0xc3, 0x08, 0x07, + 0xcb, 0x08, 0x04, 0xd3, 0x08, 0x07, 0xdb, 0x08, 0x11, 0xe3, 0x08, 0x1b, + 0xeb, 0x08, 0x34, 0xf3, 0x08, 0x1f, 0xfb, 0x08, 0x0d, 0x83, 0x09, 0x0d, + 0x8b, 0x09, 0x3c, 0x93, 0x09, 0x04, 0x9b, 0x09, 0x0e, 0xa3, 0x09, 0x04, + 0xab, 0x09, 0x3c, 0xb3, 0x09, 0x04, 0xbb, 0x09, 0x3c, 0xc3, 0x09, 0x3c, + 0xcb, 0x09, 0x04, 0xd3, 0x09, 0x1b, 0xdb, 0x09, 0x07, 0xe3, 0x09, 0x0d, + 0xeb, 0x09, 0x04, 0xf3, 0x09, 0x04, 0xfb, 0x09, 0x04, 0x83, 0x0a, 0x04, + 0x8b, 0x0a, 0x1b, 0x93, 0x0a, 0x1f, 0x9b, 0x0a, 0x11, 0xa3, 0x0a, 0x07, + 0xab, 0x0a, 0x07, 0xb3, 0x0a, 0x0d, 0xbb, 0x0a, 0x11, 0xc3, 0x0a, 0x0d, + 0xcb, 0x0a, 0x0d, 0xd3, 0x0a, 0x1b, 0xdb, 0x0a, 0x04, 0xe3, 0x0a, 0x1b, + 0xeb, 0x0a, 0x0d, 0xf3, 0x0a, 0x3c, 0xfb, 0x0a, 0x0d, 0x83, 0x0b, 0x1b, + 0x8b, 0x0b, 0x0d, 0x93, 0x0b, 0x3c, 0x9b, 0x0b, 0x3c, 0xa3, 0x0b, 0x3c, + 0xab, 0x0b, 0x07, 0xb3, 0x0b, 0x0d, 0xbb, 0x0b, 0x11, 0xc3, 0x0b, 0x07, + 0xcb, 0x0b, 0x0d, 0xd3, 0x0b, 0x0d, 0xdb, 0x0b, 0x04, 0xe3, 0x0b, 0x0d, + 0xeb, 0x0b, 0x0d, 0xf3, 0x0b, 0x0e, 0xfb, 0x0b, 0x0e, 0x83, 0x0c, 0x04, + 0x8b, 0x0c, 0x0e, 0x93, 0x0c, 0x0e, 0x9b, 0x0c, 0x0e, 0xa3, 0x0c, 0x0d, + 0xab, 0x0c, 0x0d, 0xb3, 0x0c, 0x04, 0xbb, 0x0c, 0x07, 0xc3, 0x0c, 0x63, + 0xcb, 0x0c, 0x0d, 0xd3, 0x0c, 0x0d, 0xdb, 0x0c, 0x1f, 0xe3, 0x0c, 0x3c, + 0xeb, 0x0c, 0x0d, 0xf3, 0x0c, 0x0e, 0xfb, 0x0c, 0x04, 0x83, 0x0d, 0x04, + 0x8b, 0x0d, 0x11, 0x93, 0x0d, 0x1f, 0x9b, 0x0d, 0x04, 0xa3, 0x0d, 0x0e, + 0xab, 0x0d, 0x0d, 0xb3, 0x0d, 0x0d, 0xbb, 0x0d, 0x04, 0xc3, 0x0d, 0x04, + 0xcb, 0x0d, 0x07, 0xd3, 0x0d, 0x04, 0xdb, 0x0d, 0x0d, 0xb3, 0x0e, 0x0d, + 0xbb, 0x0e, 0x04, 0xc3, 0x0e, 0x1f, 0xcb, 0x0e, 0x1b, 0xd3, 0x0e, 0x0d, + 0xdb, 0x0e, 0x07, 0xe3, 0x0e, 0x07, 0xeb, 0x0e, 0x04, 0xf3, 0x0e, 0x07, + 0xfb, 0x0e, 0x07, 0x83, 0x0f, 0x04, 0x8b, 0x0f, 0x0e, 0x93, 0x0f, 0x04, + 0x9b, 0x0f, 0x0d, 0xa3, 0x0f, 0x11, 0xab, 0x0f, 0x11, 0xb3, 0x0f, 0x3c, + 0xbb, 0x0f, 0x1f, 0xc3, 0x0f, 0x11, 0xcb, 0x0f, 0x04, 0xd3, 0x0f, 0x04, + 0xdb, 0x0f, 0x0d, 0xe3, 0x0f, 0x04, 0xeb, 0x0f, 0x3c, 0xf3, 0x0f, 0x0d, + 0xfb, 0x0f, 0x11, 0x83, 0x10, 0x0d, 0x8b, 0x10, 0x04, 0x93, 0x10, 0x11, + 0x9b, 0x10, 0x0d, 0xa3, 0x10, 0x04, 0xab, 0x10, 0x0d, 0xb3, 0x10, 0x0d, + 0xbb, 0x10, 0x04, 0xc3, 0x10, 0x07, 0xcb, 0x10, 0x07, 0xd3, 0x10, 0x04, + 0xdb, 0x10, 0x0d, 0xe3, 0x10, 0x0d, 0xeb, 0x10, 0x04, 0xf3, 0x10, 0x3c, + 0xfb, 0x10, 0x0d, 0x83, 0x11, 0x04, 0x8b, 0x11, 0x0d, 0x93, 0x11, 0x0e, + 0x9b, 0x11, 0x0e, 0xa3, 0x11, 0x0e, 0xab, 0x11, 0x0e, 0xb3, 0x11, 0x0e, + 0xbb, 0x11, 0x0e, 0xc3, 0x11, 0x15, 0xcb, 0x11, 0x07, 0xd3, 0x11, 0x0d, + 0xdb, 0x11, 0x0d, 0xe3, 0x11, 0x0d, 0xeb, 0x11, 0x3c, 0xf3, 0x11, 0x3c, + 0xfb, 0x11, 0x0d, 0x83, 0x12, 0x15, 0x8b, 0x12, 0x07, 0x93, 0x12, 0x07, + 0x9b, 0x12, 0x15, 0xa3, 0x12, 0x04, 0xab, 0x12, 0x04, 0xb3, 0x12, 0x07, + 0xbb, 0x12, 0x07, 0xc3, 0x12, 0x0e, 0xcb, 0x12, 0x0e, 0xd3, 0x12, 0x0e, + 0xdb, 0x12, 0x0d, 0xe3, 0x12, 0x3c, 0xeb, 0x12, 0x0d, 0xf3, 0x12, 0x3c, + 0xfb, 0x12, 0x0e, 0x83, 0x13, 0x15, 0x8b, 0x13, 0x15, 0x93, 0x13, 0x15, + 0x9b, 0x13, 0x0d, 0xa3, 0x13, 0x1b, 0xab, 0x13, 0x07, 0xb3, 0x13, 0x04, + 0xbb, 0x13, 0x04, 0xc3, 0x13, 0x07, 0xcb, 0x13, 0x07, 0xd3, 0x13, 0x04, + 0xdb, 0x13, 0x04, 0xe3, 0x13, 0x07, 0xeb, 0x13, 0x07, 0xf3, 0x13, 0x07, + 0xfb, 0x13, 0x07, 0x83, 0x14, 0x15, 0x8b, 0x14, 0x15, 0x93, 0x14, 0x0e, + 0x9b, 0x14, 0x04, 0xa3, 0x14, 0x04, 0xab, 0x14, 0x3c, 0xb3, 0x14, 0x04, + 0xbb, 0x14, 0x64, 0xc3, 0x14, 0x07, 0xcb, 0x14, 0x15, 0xd3, 0x14, 0x0e, + 0xdb, 0x14, 0x07, 0xe3, 0x14, 0x0e, 0xeb, 0x14, 0x0d, 0xf3, 0x14, 0x15, + 0xfb, 0x14, 0x04, 0x83, 0x15, 0x0e, 0x8b, 0x15, 0x0e, 0x93, 0x15, 0x04, + 0x9b, 0x15, 0x66, 0xa3, 0x15, 0x04, 0xab, 0x15, 0x07, 0xb3, 0x15, 0x0e, + 0xbb, 0x15, 0x07, 0xc3, 0x15, 0x07, 0xcb, 0x15, 0x07, 0xd3, 0x15, 0x07, + 0xdb, 0x15, 0x04, 0xe3, 0x15, 0x3c, 0xeb, 0x15, 0x67, 0xf3, 0x15, 0x07, + 0xfb, 0x15, 0x04, 0x83, 0x16, 0x07, 0x8b, 0x16, 0x07, 0x93, 0x16, 0x04, + 0x9b, 0x16, 0x11, 0xa3, 0x16, 0x68, 0xab, 0x16, 0x3c, 0xb3, 0x16, 0x07, + 0x00, 0x0a, 0x10, 0x00, 0x0a, 0x14, 0x00, 0x09, 0x13, 0x65, 0x00, 0x09, + 0x1a, 0x03, 0x00, 0x00, 0x09, 0x22, 0x03, 0x00, 0x0a, 0x0c, 0x00, 0x0b, + 0x6a, 0x00, 0x0b, 0x6b, 0x00, 0x0a, 0x03, 0x23, 0x6c, 0x2b, 0x6e, 0x3b, + 0x6d, 0x52, 0x02, 0x00, 0x09, 0x13, 0x6d, 0x1b, 0x6c, 0x23, 0x6e, 0x33, + 0x6d, 0x43, 0x6f, 0x4b, 0x0e, 0x51, 0x00, 0x0a, 0x07, 0x49, 0x00, 0x09, + 0x13, 0x0e, 0x29, 0x00, 0x09, 0x13, 0x66, 0x00, 0x0a, 0x04, 0x2b, 0x71, + 0x00, 0x09, 0x13, 0x71, 0x1a, 0x04, 0x00, 0x0b, 0x0e, 0x13, 0x0e, 0x1b, + 0x0e, 0x23, 0x07, 0x2b, 0x0e, 0x33, 0x07, 0x3b, 0x0e, 0x83, 0x01, 0x0e, + 0x8b, 0x01, 0x0e, 0x93, 0x01, 0x0e, 0x9b, 0x01, 0x11, 0xa3, 0x01, 0x0e, + 0xab, 0x01, 0x07, 0xb3, 0x01, 0x0e, 0xbb, 0x01, 0x04, 0xc3, 0x01, 0x07, + 0xcb, 0x01, 0x0e, 0xd3, 0x01, 0x0e, 0x00, 0x5b, 0x74, 0x63, 0x75, 0xd1, + 0x03, 0x00, 0x59, 0xf9, 0x01, 0xe9, 0x02, 0x00, 0x0b, 0x2d, 0x00, 0x89, + 0x8c, 0xb0, 0x80, 0x08}; + + MessageFilter filt; + ASSERT_TRUE( + filt.LoadFilterBytecode(kFilterBytecode, sizeof(kFilterBytecode))); + + // Pass the trace in input splitting it in slices of arbitrary size. + std::vector<MessageFilter::InputSlice> input_slices; + for (size_t i = 0; i < sizeof(kTraceData);) { + std::minstd_rand0 rnd_engine(0); + size_t slice_size = rnd_engine() % 4096; + slice_size = std::min(slice_size, sizeof(kTraceData) - i); + input_slices.emplace_back( + MessageFilter::InputSlice{kTraceData + i, slice_size}); + i += slice_size; + } + + auto filtered_data = + filt.FilterMessageFragments(input_slices.data(), input_slices.size()); + + EXPECT_GT(filtered_data.size, 0u); + EXPECT_LE(filtered_data.size, sizeof(kTraceData)); + + perfetto::protos::Trace original_trace; + ASSERT_TRUE(original_trace.ParseFromArray(kTraceData, sizeof(kTraceData))); + + perfetto::protos::Trace filtered_trace; + ASSERT_TRUE(filtered_trace.ParseFromArray( + filtered_data.data.get(), static_cast<int>(filtered_data.size))); + + // Check that the re-serialized traces are identical. + std::string original_ser = original_trace.SerializeAsString(); + std::string filter_ser = filtered_trace.SerializeAsString(); + + EXPECT_EQ(filtered_trace.packet_size(), original_trace.packet_size()); + + // Don't use EXPECT_EQ, the string is too big. If this check fails, the gtest + // diffing algorithm will take several minutes to compute the diff. + // That would mistakenly look like a CI or timing-related issue as the gtest + // would fail due to timeout. + // If this check fails, use base::HexDump() to investigate. + EXPECT_TRUE(original_ser == filter_ser); +} + +} // namespace +} // namespace protozero diff --git a/src/tracing/core/BUILD.gn b/src/tracing/core/BUILD.gn index 1c042d327..c81c7163a 100644 --- a/src/tracing/core/BUILD.gn +++ b/src/tracing/core/BUILD.gn @@ -61,6 +61,7 @@ source_set("service") { "../../../protos/perfetto/trace/perfetto:zero", # For MetatraceWriter. "../../android_stats", "../../base", + "../../protozero/filtering:message_filter", ] sources = [ "metatrace_writer.cc", @@ -74,8 +75,8 @@ source_set("service") { ] if (is_android && perfetto_build_with_android) { deps += [ - "../../android_internal:lazy_library_loader", "../../android_internal:headers", + "../../android_internal:lazy_library_loader", ] } } diff --git a/src/tracing/core/tracing_service_impl.cc b/src/tracing/core/tracing_service_impl.cc index ea17dd4ff..7084dec91 100644 --- a/src/tracing/core/tracing_service_impl.cc +++ b/src/tracing/core/tracing_service_impl.cc @@ -72,6 +72,7 @@ #include "perfetto/tracing/core/tracing_service_capabilities.h" #include "perfetto/tracing/core/tracing_service_state.h" #include "src/android_stats/statsd_logging_helper.h" +#include "src/protozero/filtering/message_filter.h" #include "src/tracing/core/packet_stream_validator.h" #include "src/tracing/core/shared_memory_arbiter_impl.h" #include "src/tracing/core/trace_buffer.h" @@ -718,6 +719,35 @@ base::Status TracingServiceImpl::EnableTracing(ConsumerEndpointImpl* consumer, tracing_sessions_.size()); } + // If the trace config provides a filter bytecode, setup the filter now. + // If the filter loading fails, abort the tracing session rather than running + // unfiltered. + std::unique_ptr<protozero::MessageFilter> trace_filter; + if (cfg.has_trace_filter()) { + const auto& filt = cfg.trace_filter(); + const std::string& bytecode = filt.bytecode(); + trace_filter.reset(new protozero::MessageFilter()); + if (!trace_filter->LoadFilterBytecode(bytecode.data(), bytecode.size())) { + MaybeLogUploadEvent( + cfg, PerfettoStatsdAtom::kTracedEnableTracingInvalidFilter); + return PERFETTO_SVC_ERR("Trace filter bytecode invalid, aborting"); + } + // The filter is created using perfetto.protos.Trace as root message + // (because that makes it possible to play around with the `proto_filter` + // tool on actual traces). Here in the service, however, we deal with + // perfetto.protos.TracePacket(s), which are one level down (Trace.packet). + // The IPC client (or the write_into_filte logic in here) are responsible + // for pre-pending the packet preamble (See GetProtoPreamble() calls), but + // the preamble is not there at ReadBuffer time. Hence we change the root of + // the filtering to start at the Trace.packet level. + uint32_t packet_field_id = TracePacket::kPacketFieldNumber; + if (!trace_filter->SetFilterRoot(&packet_field_id, 1)) { + MaybeLogUploadEvent( + cfg, PerfettoStatsdAtom::kTracedEnableTracingInvalidFilter); + return PERFETTO_SVC_ERR("Failed to set filter root."); + } + } + const TracingSessionID tsid = ++last_tracing_session_id_; TracingSession* tracing_session = &tracing_sessions_ @@ -725,6 +755,9 @@ base::Status TracingServiceImpl::EnableTracing(ConsumerEndpointImpl* consumer, std::forward_as_tuple(tsid, consumer, cfg, task_runner_)) .first->second; + if (trace_filter) + tracing_session->trace_filter = std::move(trace_filter); + if (cfg.write_into_file()) { if (!fd ^ !cfg.output_path().empty()) { tracing_sessions_.erase(tsid); @@ -2073,6 +2106,48 @@ bool TracingServiceImpl::ReadBuffers(TracingSessionID tsid, total_slices += packets[i].slices().size(); } + // +-------------------------------------------------------------------------+ + // | NO MORE CHANGES TO |packets| AFTER THIS POINT. | + // +-------------------------------------------------------------------------+ + + // If the tracing session specified a filter, run all packets through the + // filter and replace them with the filter results. + // The process below mantains the cardinality of input packets. Even if an + // entire packet is filtered out, we emit a zero-sized TracePacket proto. That + // makes debugging and reasoning about the trace stats easier. + // This place swaps the contents of each |packets| entry in place. + if (tracing_session->trace_filter) { + auto& trace_filter = *tracing_session->trace_filter; + // The filter root shoud be reset from protos.Trace to protos.TracePacket + // by the earlier call to SetFilterRoot() in EnableTracing(). + PERFETTO_DCHECK(trace_filter.root_msg_index() != 0); + std::vector<protozero::MessageFilter::InputSlice> filter_input; + for (auto it = packets.begin(); it != packets.end(); ++it) { + const auto& packet_slices = it->slices(); + filter_input.clear(); + filter_input.resize(packet_slices.size()); + ++tracing_session->filter_input_packets; + tracing_session->filter_input_bytes += it->size(); + for (size_t i = 0; i < packet_slices.size(); ++i) + filter_input[i] = {packet_slices[i].start, packet_slices[i].size}; + auto filtered_packet = trace_filter.FilterMessageFragments( + &filter_input[0], filter_input.size()); + + // Replace the packet in-place with the filtered one (unless failed). + *it = TracePacket(); + if (filtered_packet.error) { + ++tracing_session->filter_errors; + PERFETTO_DLOG("Trace packet filtering failed @ packet %" PRIu64, + tracing_session->filter_input_packets); + continue; + } + tracing_session->filter_output_bytes += filtered_packet.size; + it->AddSlice(Slice::TakeOwnership(std::move(filtered_packet.data), + filtered_packet.size)); + + } // for (packet) + } // if (trace_filter) + // If the caller asked us to write into a file by setting // |write_into_file| == true in the trace config, drain the packets read // (if any) into the given file descriptor. @@ -2924,6 +2999,14 @@ TraceStats TracingServiceImpl::GetTraceStats(TracingSession* tracing_session) { trace_stats.set_patches_discarded(patches_discarded_); trace_stats.set_invalid_packets(tracing_session->invalid_packets); + if (tracing_session->trace_filter) { + auto* filt_stats = trace_stats.mutable_filter_stats(); + filt_stats->set_input_packets(tracing_session->filter_input_packets); + filt_stats->set_input_bytes(tracing_session->filter_input_bytes); + filt_stats->set_output_bytes(tracing_session->filter_output_bytes); + filt_stats->set_errors(tracing_session->filter_errors); + } + for (BufferID buf_id : tracing_session->buffers_index) { TraceBuffer* buf = GetBufferByID(buf_id); if (!buf) { diff --git a/src/tracing/core/tracing_service_impl.h b/src/tracing/core/tracing_service_impl.h index e97ba2a1b..27b172cec 100644 --- a/src/tracing/core/tracing_service_impl.h +++ b/src/tracing/core/tracing_service_impl.h @@ -47,6 +47,10 @@ #include "src/android_stats/perfetto_atoms.h" #include "src/tracing/core/id_allocator.h" +namespace protozero { +class MessageFilter; +} + namespace perfetto { namespace base { @@ -574,6 +578,13 @@ class TracingServiceImpl : public TracingService { // Periodic task for snapshotting service events (e.g. clocks, sync markers // etc) base::PeriodicTask snapshot_periodic_task; + + // When non-NULL the packets should be post-processed using the filter. + std::unique_ptr<protozero::MessageFilter> trace_filter; + uint64_t filter_input_packets = 0; + uint64_t filter_input_bytes = 0; + uint64_t filter_output_bytes = 0; + uint64_t filter_errors = 0; }; TracingServiceImpl(const TracingServiceImpl&) = delete; diff --git a/test/configs/BUILD.gn b/test/configs/BUILD.gn index 4f77441c8..4c93db1ab 100644 --- a/test/configs/BUILD.gn +++ b/test/configs/BUILD.gn @@ -35,11 +35,16 @@ action_foreach("configs") { "client_api.cfg", "ftrace.cfg", "ftrace_largebuffer.cfg", + "ftrace_with_filter.cfg", + "ftrace_with_ksyms.cfg", "heapprofd.cfg", "long_trace.cfg", + "mm_events.cfg", "scheduling.cfg", "summary.cfg", "sys_stats.cfg", + "thermal.cfg", + "traced_perf.cfg", ] outputs = [ "$root_out_dir/{{source_file_part}}.protobuf" ] diff --git a/test/configs/ftrace_with_filter.cfg b/test/configs/ftrace_with_filter.cfg new file mode 100644 index 000000000..54c316171 --- /dev/null +++ b/test/configs/ftrace_with_filter.cfg @@ -0,0 +1,36 @@ +buffers { + size_kb: 65536 +} + +data_sources { + config { + name: "linux.ftrace" + target_buffer: 0 + ftrace_config { + ftrace_events: "sched/sched_process_exec" + ftrace_events: "sched/sched_process_exit" + ftrace_events: "sched/sched_process_fork" + ftrace_events: "sched/sched_process_free" + ftrace_events: "sched/sched_process_hang" + ftrace_events: "sched/sched_process_wait" + ftrace_events: "sched/sched_switch" + ftrace_events: "sched/sched_wakeup_new" + ftrace_events: "sched/sched_wakeup" + ftrace_events: "sched/sched_waking" + } + } +} + +data_sources { + config { + name: "linux.process_stats" + target_buffer: 0 + } +} + +trace_filter { + # A very minimal bytecode which allows only sched_switch and nothing more. + bytecode: "\013\001\000\013\002\101\121\151\321\002\000\011\023\003\031\000\012\002\043\004\000\012\007\000\273\341\337\347\016" +} + +duration_ms: 10000 diff --git a/tools/BUILD.gn b/tools/BUILD.gn index a8342f934..772f6a239 100644 --- a/tools/BUILD.gn +++ b/tools/BUILD.gn @@ -24,6 +24,7 @@ group("tools") { ":copy_protoc", "compact_reencode", "ftrace_proto_gen", + "proto_filter", "protoprofile", ] if (is_linux || is_android) { diff --git a/tools/install-build-deps b/tools/install-build-deps index bc7765eec..878f1f683 100755 --- a/tools/install-build-deps +++ b/tools/install-build-deps @@ -234,8 +234,8 @@ BUILD_DEPS_HOST = [ # Example traces for regression tests. Dependency( 'test/data.zip', - 'https://storage.googleapis.com/perfetto/test-data-20210416-122153.zip', - '0c3a7b353c1100b783873009949596e75db5ecc4a5cf114f24ccf3f1b08570d3', + 'https://storage.googleapis.com/perfetto/test-data-20210513-224349.zip', + '3dcc146f4ce38d17fd1f8c4c65af07e7cf7c5c4cb8aa4c7bf73ec3a095d997d1', 'all', 'all', ), diff --git a/tools/proto_filter/BUILD.gn b/tools/proto_filter/BUILD.gn new file mode 100644 index 000000000..b2b6997a3 --- /dev/null +++ b/tools/proto_filter/BUILD.gn @@ -0,0 +1,29 @@ +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("../../gn/perfetto_host_executable.gni") + +perfetto_host_executable("proto_filter") { + testonly = true + deps = [ + "../../gn:default_deps", + "../../gn:protobuf_full", + "../../src/base", + "../../src/protozero", + "../../src/protozero/filtering:bytecode_generator", + "../../src/protozero/filtering:filter_util", + "../../src/protozero/filtering:message_filter", + ] + sources = [ "proto_filter.cc" ] +} diff --git a/tools/proto_filter/proto_filter.cc b/tools/proto_filter/proto_filter.cc new file mode 100644 index 000000000..e7443cb99 --- /dev/null +++ b/tools/proto_filter/proto_filter.cc @@ -0,0 +1,294 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "perfetto/ext/base/file_utils.h" +#include "perfetto/ext/base/getopt.h" +#include "perfetto/ext/base/scoped_file.h" +#include "perfetto/ext/base/string_utils.h" +#include "perfetto/ext/base/version.h" +#include "src/protozero/filtering/filter_util.h" +#include "src/protozero/filtering/message_filter.h" + +namespace perfetto { +namespace proto_filter { +namespace { + +const char kUsage[] = + R"(Usage: proto_filter [-s schema_in] [-i message in] [-o message out] [-f filter in] [-F filter out] [-T filter_oct_out] [-d --dedupe] [-I proto include path] [-r root message] + +-s --schema-in: Path to the root .proto file. Required for most operations +-I --proto_path: Extra include directory for proto includes. If omitted assumed CWD. +-r --root_message: Fully qualified name for the root proto message (e.g. perfetto.protos.Trace) + If omitted the first message defined in the schema will be used. +-i --msg_in: Path of a binary-encoded proto message which will be filtered. +-o --msg_out: Path of the binary-encoded filtered proto message written in output. +-f --filter_in: Path of a filter bytecode file previously generated by this tool. +-F --filter_out: Path of the filter bytecode file generated from the --schema-in definition. +-T --filter_oct_out: Like --filter_out, but emits a octal-escaped C string suitable for .pbtx. +-d --dedupe: Minimize filter size by deduping leaf messages with same field ids. + +Example usage: + +# Convert a .proto schema file into a diff-friendly list of messages/fields> + + proto_filter -r perfetto.protos.Trace -s protos/perfetto/trace/trace.proto + +# Generate the filter bytecode from a .proto schema + + proto_filter -r perfetto.protos.Trace -s protos/perfetto/trace/trace.proto \ + -F /tmp/bytecode [--dedupe] + +# List the used/filtered fields from a trace file + + proto_filter -r perfetto.protos.Trace -s protos/perfetto/trace/trace.proto \ + -i test/data/example_android_trace_30s.pb -f /tmp/bytecode + +# Filter a trace using a filter bytecode + + proto_filter -i test/data/example_android_trace_30s.pb -f /tmp/bytecode \ + -o /tmp/filtered_trace +)"; + +int Main(int argc, char** argv) { + static const option long_options[] = { + {"help", no_argument, nullptr, 'h'}, + {"version", no_argument, nullptr, 'v'}, + {"dedupe", no_argument, nullptr, 'd'}, + {"proto_path", no_argument, nullptr, 'I'}, + {"schema_in", no_argument, nullptr, 's'}, + {"root_message", no_argument, nullptr, 'r'}, + {"msg_in", no_argument, nullptr, 'i'}, + {"msg_out", no_argument, nullptr, 'o'}, + {"filter_in", no_argument, nullptr, 'f'}, + {"filter_out", no_argument, nullptr, 'F'}, + {"filter_oct_out", no_argument, nullptr, 'T'}, + {nullptr, 0, nullptr, 0}}; + + std::string msg_in; + std::string msg_out; + std::string filter_in; + std::string schema_in; + std::string filter_out; + std::string filter_oct_out; + std::string proto_path; + std::string root_message_arg; + bool dedupe = false; + + for (;;) { + int option = + getopt_long(argc, argv, "hvdI:s:r:i:o:f:F:T:", long_options, nullptr); + + if (option == -1) + break; // EOF. + + if (option == 'v') { + printf("%s\n", base::GetVersionString()); + exit(0); + } + + if (option == 'd') { + dedupe = true; + continue; + } + + if (option == 'I') { + proto_path = optarg; + continue; + } + + if (option == 's') { + schema_in = optarg; + continue; + } + + if (option == 'r') { + root_message_arg = optarg; + continue; + } + + if (option == 'i') { + msg_in = optarg; + continue; + } + + if (option == 'o') { + msg_out = optarg; + continue; + } + + if (option == 'f') { + filter_in = optarg; + continue; + } + + if (option == 'F') { + filter_out = optarg; + continue; + } + + if (option == 'T') { + filter_oct_out = optarg; + continue; + } + + if (option == 'h') { + fprintf(stdout, kUsage); + exit(0); + } + + fprintf(stderr, kUsage); + exit(1); + } + + if (msg_in.empty() && filter_in.empty() && schema_in.empty()) { + fprintf(stderr, kUsage); + return 1; + } + + std::string msg_in_data; + if (!msg_in.empty()) { + PERFETTO_LOG("Loading proto-encoded message from %s", msg_in.c_str()); + if (!base::ReadFile(msg_in, &msg_in_data)) { + PERFETTO_ELOG("Could not open message file %s", msg_in.c_str()); + return 1; + } + } + + protozero::FilterUtil filter; + if (!schema_in.empty()) { + PERFETTO_LOG("Loading proto schema from %s", schema_in.c_str()); + if (!filter.LoadMessageDefinition(schema_in, root_message_arg, + proto_path)) { + PERFETTO_ELOG("Failed to parse proto schema from %s", schema_in.c_str()); + return 1; + } + if (dedupe) + filter.Dedupe(); + } + + protozero::MessageFilter msg_filter; + std::string filter_data; + std::string filter_data_src; + if (!filter_in.empty()) { + PERFETTO_LOG("Loading filter bytecode from %s", filter_in.c_str()); + if (!base::ReadFile(filter_in, &filter_data)) { + PERFETTO_ELOG("Could not open filter file %s", filter_in.c_str()); + return 1; + } + filter_data_src = filter_in; + } else if (!schema_in.empty()) { + PERFETTO_LOG("Generating filter bytecode from %s", schema_in.c_str()); + filter_data = filter.GenerateFilterBytecode(); + filter_data_src = schema_in; + } + + if (!filter_data.empty()) { + const uint8_t* data = reinterpret_cast<const uint8_t*>(filter_data.data()); + if (!msg_filter.LoadFilterBytecode(data, filter_data.size())) { + PERFETTO_ELOG("Failed to parse filter bytecode from %s", + filter_data_src.c_str()); + return 1; + } + } + + // Write the filter bytecode in output. + if (!filter_out.empty()) { + auto fd = base::OpenFile(filter_out, O_WRONLY | O_TRUNC | O_CREAT, 0644); + if (!fd) { + PERFETTO_ELOG("Could not open filter out path %s", filter_out.c_str()); + return 1; + } + PERFETTO_LOG("Writing filter bytecode (%zu bytes) into %s", + filter_data.size(), filter_out.c_str()); + base::WriteAll(*fd, filter_data.data(), filter_data.size()); + } + + if (!filter_oct_out.empty()) { + auto fd = + base::OpenFile(filter_oct_out, O_WRONLY | O_TRUNC | O_CREAT, 0644); + if (!fd) { + PERFETTO_ELOG("Could not open filter out path %s", + filter_oct_out.c_str()); + return 1; + } + std::string oct_str; + oct_str.reserve(filter_data.size() * 4 + 64); + oct_str.append("trace_filter{\n bytecode: \""); + for (char c : filter_data) { + uint8_t octect = static_cast<uint8_t>(c); + char buf[5]{'\\', '0', '0', '0', 0}; + for (uint8_t i = 0; i < 3; ++i) { + buf[3 - i] = static_cast<char>('0' + static_cast<uint8_t>(octect) % 8); + octect /= 8; + } + oct_str.append(buf); + } + oct_str.append("\"\n}\n"); + PERFETTO_LOG("Writing filter bytecode (%zu bytes) into %s", oct_str.size(), + filter_oct_out.c_str()); + base::WriteAll(*fd, oct_str.data(), oct_str.size()); + } + + // Apply the filter to the input message (if any). + std::vector<uint8_t> msg_filtered_data; + if (!msg_in.empty()) { + PERFETTO_LOG("Applying filter %s to proto message %s", + filter_data_src.c_str(), msg_in.c_str()); + msg_filter.enable_field_usage_tracking(true); + auto res = msg_filter.FilterMessage(msg_in_data.data(), msg_in_data.size()); + if (res.error) + PERFETTO_FATAL("Filtering failed"); + msg_filtered_data.insert(msg_filtered_data.end(), res.data.get(), + res.data.get() + res.size); + } + + // Write out the filtered message. + if (!msg_out.empty()) { + PERFETTO_LOG("Writing filtered proto bytes (%zu bytes) into %s", + msg_filtered_data.size(), msg_out.c_str()); + auto fd = base::OpenFile(msg_out, O_WRONLY | O_CREAT, 0644); + base::WriteAll(*fd, msg_filtered_data.data(), msg_filtered_data.size()); + } + + if (!msg_in.empty()) { + const auto& field_usage_map = msg_filter.field_usage(); + for (const auto& it : field_usage_map) { + const std::string& field_path_varint = it.first; + int32_t num_occurrences = it.second; + std::string path_str = filter.LookupField(field_path_varint); + printf("%-100s %s %d\n", path_str.c_str(), + num_occurrences < 0 ? "DROP" : "PASS", std::abs(num_occurrences)); + } + } else if (!schema_in.empty()) { + filter.PrintAsText(); + } + + if ((!filter_out.empty() || !filter_oct_out.empty()) && !dedupe) { + PERFETTO_ELOG( + "Warning: looks like you are generating a filter without --dedupe. For " + "production use cases, --dedupe can make the output bytecode " + "significantly smaller."); + } + return 0; +} + +} // namespace +} // namespace proto_filter +} // namespace perfetto + +int main(int argc, char** argv) { + return perfetto::proto_filter::Main(argc, argv); +} diff --git a/tools/run_android_test b/tools/run_android_test index 64cc560cf..5f3895171 100755 --- a/tools/run_android_test +++ b/tools/run_android_test @@ -143,8 +143,10 @@ def Main(): AdbCall('root') AdbCall('wait-for-device') - target_dir = '/data/local/tmp/' + args.test_name - AdbCall('shell', 'rm -rf "%s"; mkdir -p "%s"' % (2 * (target_dir,))) + target_dir = '/data/local/tmp/perfetto_tests' + if not args.no_cleanup: + AdbCall('shell', 'rm -rf "%s"' % target_dir) + AdbCall('shell', 'mkdir -p "%s"' % target_dir) # Some tests require the trace directory to exist, while true for android # devices in general some emulators might not have it set up. So we check to # see if it exists, and if not create it. diff --git a/tools/test_data.txt b/tools/test_data.txt index abcb65d30..55ee848de 100644 --- a/tools/test_data.txt +++ b/tools/test_data.txt @@ -1,5 +1,8 @@ # List of test deps that should be pushed on the device. Paths are relative # to the root. -src/traced/probes/ftrace/test/data/ src/traced/probes/filesystem/testdata/ +src/traced/probes/ftrace/test/data/ +test/data/android_log_ring_buffer_mode.pb +test/data/example_android_trace_30s.pb +test/data/full_trace_filter.bytecode test/data/kallsyms.txt |