diff options
Diffstat (limited to 'src/trace_processor/importers')
100 files changed, 4117 insertions, 2208 deletions
diff --git a/src/trace_processor/importers/android_bugreport/android_bugreport_parser.cc b/src/trace_processor/importers/android_bugreport/android_bugreport_parser.cc index 87aa0f5f9..1d46efc82 100644 --- a/src/trace_processor/importers/android_bugreport/android_bugreport_parser.cc +++ b/src/trace_processor/importers/android_bugreport/android_bugreport_parser.cc @@ -17,54 +17,90 @@ #include "src/trace_processor/importers/android_bugreport/android_bugreport_parser.h" #include <algorithm> +#include <cstddef> #include <optional> +#include <string> +#include <vector> #include "perfetto/base/logging.h" +#include "perfetto/base/status.h" #include "perfetto/ext/base/string_utils.h" -#include "perfetto/trace_processor/trace_blob.h" -#include "perfetto/trace_processor/trace_blob_view.h" +#include "protos/perfetto/common/builtin_clock.pbzero.h" #include "src/trace_processor/importers/android_bugreport/android_log_parser.h" #include "src/trace_processor/importers/common/clock_tracker.h" #include "src/trace_processor/importers/common/process_tracker.h" -#include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/util/zip_reader.h" -#include "protos/perfetto/common/builtin_clock.pbzero.h" - namespace perfetto { namespace trace_processor { +namespace { +const util::ZipFile* FindBugReportFile( + const std::vector<util::ZipFile>& zip_file_entries) { + for (const auto& zf : zip_file_entries) { + if (base::StartsWith(zf.name(), "bugreport-") && + base::EndsWith(zf.name(), ".txt")) { + return &zf; + } + } + return nullptr; +} -AndroidBugreportParser::AndroidBugreportParser(TraceProcessorContext* ctx) - : context_(ctx), zip_reader_(new util::ZipReader()) {} +std::optional<int32_t> ExtractYearFromBugReportFilename( + const std::string& filename) { + // Typical name: "bugreport-product-TP1A.220623.001-2022-06-24-16-24-37.txt". + auto year_str = + filename.substr(filename.size() - strlen("2022-12-31-23-59-00.txt"), 4); + return base::StringToInt32(year_str); +} -AndroidBugreportParser::~AndroidBugreportParser() = default; +} // namespace -util::Status AndroidBugreportParser::Parse(TraceBlobView tbv) { - if (!first_chunk_seen_) { - first_chunk_seen_ = true; - // All logs in Android bugreports use wall time (which creates problems - // in case of early boot events before NTP kicks in, which get emitted as - // 1970), but that is the state of affairs. - context_->clock_tracker->SetTraceTimeClock( - protos::pbzero::BUILTIN_CLOCK_REALTIME); +// static +bool AndroidBugreportParser::IsAndroidBugReport( + const std::vector<util::ZipFile>& zip_file_entries) { + if (const util::ZipFile* file = FindBugReportFile(zip_file_entries); + file != nullptr) { + return ExtractYearFromBugReportFilename(file->name()).has_value(); } - return zip_reader_->Parse(tbv.data(), tbv.size()); + return false; } -void AndroidBugreportParser::NotifyEndOfFile() { +// static +util::Status AndroidBugreportParser::Parse( + TraceProcessorContext* context, + std::vector<util::ZipFile> zip_file_entries) { + return AndroidBugreportParser(context, std::move(zip_file_entries)) + .ParseImpl(); +} + +AndroidBugreportParser::AndroidBugreportParser( + TraceProcessorContext* context, + std::vector<util::ZipFile> zip_file_entries) + : context_(context), zip_file_entries_(std::move(zip_file_entries)) {} + +AndroidBugreportParser::~AndroidBugreportParser() = default; + +util::Status AndroidBugreportParser::ParseImpl() { + // All logs in Android bugreports use wall time (which creates problems + // in case of early boot events before NTP kicks in, which get emitted as + // 1970), but that is the state of affairs. + context_->clock_tracker->SetTraceTimeClock( + protos::pbzero::BUILTIN_CLOCK_REALTIME); if (!DetectYearAndBrFilename()) { context_->storage->IncrementStats(stats::android_br_parse_errors); - return; + return base::ErrStatus("Zip file does not contain bugreport file."); } ParsePersistentLogcat(); ParseDumpstateTxt(); SortAndStoreLogcat(); + return base::OkStatus(); } void AndroidBugreportParser::ParseDumpstateTxt() { + PERFETTO_CHECK(dumpstate_file_); // Dumpstate is organized in a two level hierarchy, beautifully flattened into // one text file with load bearing ----- markers: // 1. Various dumpstate sections, examples: @@ -95,80 +131,81 @@ void AndroidBugreportParser::ParseDumpstateTxt() { // Here we put each line in a dedicated table, android_dumpstate, keeping // track of the dumpstate `section` and dumpsys `service`. AndroidLogParser log_parser(br_year_, context_->storage.get()); - util::ZipFile* zf = zip_reader_->Find(dumpstate_fname_); StringId section_id = StringId::Null(); // The current dumpstate section. StringId service_id = StringId::Null(); // The current dumpsys service. static constexpr size_t npos = base::StringView::npos; enum { OTHER = 0, DUMPSYS, LOG } cur_sect = OTHER; - zf->DecompressLines([&](const std::vector<base::StringView>& lines) { - // Optimization for ParseLogLines() below. Avoids ctor/dtor-ing a new vector - // on every line. - std::vector<base::StringView> log_line(1); - for (const base::StringView& line : lines) { - if (line.StartsWith("------ ") && line.EndsWith(" ------")) { - // These lines mark the beginning and end of dumpstate sections: - // ------ DUMPSYS CRITICAL (/system/bin/dumpsys) ------ - // ------ 0.356s was the duration of 'DUMPSYS CRITICAL' ------ - base::StringView section = line.substr(7); - section = section.substr(0, section.size() - 7); - bool end_marker = section.find("was the duration of") != npos; - service_id = StringId::Null(); - if (end_marker) { - section_id = StringId::Null(); - } else { - section_id = context_->storage->InternString(section); - cur_sect = OTHER; - if (section.StartsWith("DUMPSYS")) { - cur_sect = DUMPSYS; - } else if (section.StartsWith("SYSTEM LOG") || - section.StartsWith("EVENT LOG") || - section.StartsWith("RADIO LOG")) { - // KERNEL LOG is deliberately omitted because SYSTEM LOG is a - // superset. KERNEL LOG contains all dupes. - cur_sect = LOG; - } else if (section.StartsWith("BLOCK STAT")) { - // Coalesce all the block stats into one section. Otherwise they - // pollute the table with one section per block device. - section_id = context_->storage->InternString("BLOCK STAT"); + dumpstate_file_->DecompressLines( + [&](const std::vector<base::StringView>& lines) { + // Optimization for ParseLogLines() below. Avoids ctor/dtor-ing a new + // vector on every line. + std::vector<base::StringView> log_line(1); + for (const base::StringView& line : lines) { + if (line.StartsWith("------ ") && line.EndsWith(" ------")) { + // These lines mark the beginning and end of dumpstate sections: + // ------ DUMPSYS CRITICAL (/system/bin/dumpsys) ------ + // ------ 0.356s was the duration of 'DUMPSYS CRITICAL' ------ + base::StringView section = line.substr(7); + section = section.substr(0, section.size() - 7); + bool end_marker = section.find("was the duration of") != npos; + service_id = StringId::Null(); + if (end_marker) { + section_id = StringId::Null(); + } else { + section_id = context_->storage->InternString(section); + cur_sect = OTHER; + if (section.StartsWith("DUMPSYS")) { + cur_sect = DUMPSYS; + } else if (section.StartsWith("SYSTEM LOG") || + section.StartsWith("EVENT LOG") || + section.StartsWith("RADIO LOG")) { + // KERNEL LOG is deliberately omitted because SYSTEM LOG is a + // superset. KERNEL LOG contains all dupes. + cur_sect = LOG; + } else if (section.StartsWith("BLOCK STAT")) { + // Coalesce all the block stats into one section. Otherwise they + // pollute the table with one section per block device. + section_id = context_->storage->InternString("BLOCK STAT"); + } + } + continue; + } + // Skip end marker lines for dumpsys sections. + if (cur_sect == DUMPSYS && line.StartsWith("--------- ") && + line.find("was the duration of dumpsys") != npos) { + service_id = StringId::Null(); + continue; + } + if (cur_sect == DUMPSYS && service_id.is_null() && + line.StartsWith( + "----------------------------------------------")) { + continue; + } + if (cur_sect == DUMPSYS && line.StartsWith("DUMP OF SERVICE")) { + // DUMP OF SERVICE [CRITICAL|HIGH] ServiceName: + base::StringView svc = line.substr(line.rfind(' ') + 1); + svc = svc.substr(0, svc.size() - 1); + service_id = context_->storage->InternString(svc); + } else if (cur_sect == LOG) { + // Parse the non-persistent logcat and append to `log_events_`, + // together with the persistent one previously parsed by + // ParsePersistentLogcat(). Skips entries that are already seen in + // the persistent logcat, handling us vs ms truncation. + PERFETTO_DCHECK(log_line.size() == 1); + log_line[0] = line; + log_parser.ParseLogLines(log_line, &log_events_, + log_events_last_sorted_idx_); } - } - continue; - } - // Skip end marker lines for dumpsys sections. - if (cur_sect == DUMPSYS && line.StartsWith("--------- ") && - line.find("was the duration of dumpsys") != npos) { - service_id = StringId::Null(); - continue; - } - if (cur_sect == DUMPSYS && service_id.is_null() && - line.StartsWith("----------------------------------------------")) { - continue; - } - if (cur_sect == DUMPSYS && line.StartsWith("DUMP OF SERVICE")) { - // DUMP OF SERVICE [CRITICAL|HIGH] ServiceName: - base::StringView svc = line.substr(line.rfind(' ') + 1); - svc = svc.substr(0, svc.size() - 1); - service_id = context_->storage->InternString(svc); - } else if (cur_sect == LOG) { - // Parse the non-persistent logcat and append to `log_events_`, together - // with the persistent one previously parsed by ParsePersistentLogcat(). - // Skips entries that are already seen in the persistent logcat, - // handling us vs ms truncation. - PERFETTO_DCHECK(log_line.size() == 1); - log_line[0] = line; - log_parser.ParseLogLines(log_line, &log_events_, - log_events_last_sorted_idx_); - } - if (build_fpr_.empty() && line.StartsWith("Build fingerprint:")) { - build_fpr_ = line.substr(20, line.size() - 20).ToStdString(); - } + if (build_fpr_.empty() && line.StartsWith("Build fingerprint:")) { + build_fpr_ = line.substr(20, line.size() - 20).ToStdString(); + } - // Append the line to the android_dumpstate table. - context_->storage->mutable_android_dumpstate_table()->Insert( - {section_id, service_id, context_->storage->InternString(line)}); - } - }); + // Append the line to the android_dumpstate table. + context_->storage->mutable_android_dumpstate_table()->Insert( + {section_id, service_id, context_->storage->InternString(line)}); + } + }); } void AndroidBugreportParser::ParsePersistentLogcat() { @@ -182,22 +219,23 @@ void AndroidBugreportParser::ParsePersistentLogcat() { // Sort files to ease the job of the subsequent line-based sort. Unfortunately // lines within each file are not 100% timestamp-ordered, due to things like // kernel messages where log time != event time. - std::vector<std::pair<uint64_t, std::string>> log_paths; - for (const util::ZipFile& zf : zip_reader_->files()) { + std::vector<std::pair<uint64_t, const util::ZipFile*>> log_files; + for (const util::ZipFile& zf : zip_file_entries_) { if (base::StartsWith(zf.name(), "FS/data/misc/logd/logcat") && !base::EndsWith(zf.name(), "logcat.id")) { - log_paths.emplace_back(std::make_pair(zf.GetDatetime(), zf.name())); + log_files.push_back(std::make_pair(zf.GetDatetime(), &zf)); } } - std::sort(log_paths.begin(), log_paths.end()); + + std::sort(log_files.begin(), log_files.end()); // Push all events into the AndroidLogParser. It will take care of string // interning into the pool. Appends entries into `log_events`. - for (const auto& kv : log_paths) { - util::ZipFile* zf = zip_reader_->Find(kv.second); - zf->DecompressLines([&](const std::vector<base::StringView>& lines) { - log_parser.ParseLogLines(lines, &log_events_); - }); + for (const auto& log_file : log_files) { + log_file.second->DecompressLines( + [&](const std::vector<base::StringView>& lines) { + log_parser.ParseLogLines(lines, &log_events_); + }); } // Do an initial sorting pass. This is not the final sorting because we @@ -230,30 +268,20 @@ void AndroidBugreportParser::SortAndStoreLogcat() { // This is obviously bugged for cases of bugreports collected across new year // but we'll live with that. bool AndroidBugreportParser::DetectYearAndBrFilename() { - const util::ZipFile* br_file = nullptr; - for (const auto& zf : zip_reader_->files()) { - if (base::StartsWith(zf.name(), "bugreport-") && - base::EndsWith(zf.name(), ".txt")) { - br_file = &zf; - break; - } - } - + const util::ZipFile* br_file = FindBugReportFile(zip_file_entries_); if (!br_file) { PERFETTO_ELOG("Could not find bugreport-*.txt in the zip file"); return false; } - // Typical name: "bugreport-product-TP1A.220623.001-2022-06-24-16-24-37.txt". - auto year_str = br_file->name().substr( - br_file->name().size() - strlen("2022-12-31-23-59-00.txt"), 4); - std::optional<int32_t> year = base::StringToInt32(year_str); + std::optional<int32_t> year = + ExtractYearFromBugReportFilename(br_file->name()); if (!year.has_value()) { PERFETTO_ELOG("Could not parse the year from %s", br_file->name().c_str()); return false; } br_year_ = *year; - dumpstate_fname_ = br_file->name(); + dumpstate_file_ = br_file; return true; } diff --git a/src/trace_processor/importers/android_bugreport/android_bugreport_parser.h b/src/trace_processor/importers/android_bugreport/android_bugreport_parser.h index 99691598f..a9ca3226c 100644 --- a/src/trace_processor/importers/android_bugreport/android_bugreport_parser.h +++ b/src/trace_processor/importers/android_bugreport/android_bugreport_parser.h @@ -17,8 +17,11 @@ #ifndef SRC_TRACE_PROCESSOR_IMPORTERS_ANDROID_BUGREPORT_ANDROID_BUGREPORT_PARSER_H_ #define SRC_TRACE_PROCESSOR_IMPORTERS_ANDROID_BUGREPORT_ANDROID_BUGREPORT_PARSER_H_ -#include "src/trace_processor/importers/common/chunked_trace_reader.h" -#include "src/trace_processor/storage/trace_storage.h" +#include <cstddef> +#include <vector> + +#include "perfetto/trace_processor/status.h" +#include "src/trace_processor/util/zip_reader.h" namespace perfetto { namespace trace_processor { @@ -31,16 +34,19 @@ struct AndroidLogEvent; class TraceProcessorContext; // Trace importer for Android bugreport.zip archives. -class AndroidBugreportParser : public ChunkedTraceReader { +class AndroidBugreportParser { public: - explicit AndroidBugreportParser(TraceProcessorContext*); - ~AndroidBugreportParser() override; - - // ChunkedTraceReader implementation. - util::Status Parse(TraceBlobView) override; - void NotifyEndOfFile() override; + static bool IsAndroidBugReport( + const std::vector<util::ZipFile>& zip_file_entries); + static util::Status Parse(TraceProcessorContext* context, + std::vector<util::ZipFile> zip_file_entries); private: + AndroidBugreportParser(TraceProcessorContext* context, + std::vector<util::ZipFile> zip_file_entries); + ~AndroidBugreportParser(); + util::Status ParseImpl(); + bool DetectYearAndBrFilename(); void ParsePersistentLogcat(); void ParseDumpstateTxt(); @@ -48,11 +54,11 @@ class AndroidBugreportParser : public ChunkedTraceReader { void SortLogEvents(); TraceProcessorContext* const context_; + std::vector<util::ZipFile> zip_file_entries_; int br_year_ = 0; // The year when the bugreport has been taken. - std::string dumpstate_fname_; // The name of bugreport-xxx-2022-08-04....txt + const util::ZipFile* dumpstate_file_ = + nullptr; // The bugreport-xxx-2022-08-04....txt file std::string build_fpr_; - bool first_chunk_seen_ = false; - std::unique_ptr<util::ZipReader> zip_reader_; std::vector<AndroidLogEvent> log_events_; size_t log_events_last_sorted_idx_ = 0; }; diff --git a/src/trace_processor/importers/common/BUILD.gn b/src/trace_processor/importers/common/BUILD.gn index 43dfb2345..9cde6e32d 100644 --- a/src/trace_processor/importers/common/BUILD.gn +++ b/src/trace_processor/importers/common/BUILD.gn @@ -28,6 +28,8 @@ source_set("common") { "clock_converter.h", "clock_tracker.cc", "clock_tracker.h", + "cpu_tracker.cc", + "cpu_tracker.h", "create_mapping_params.h", "deobfuscation_mapping_table.cc", "deobfuscation_mapping_table.h", @@ -45,6 +47,8 @@ source_set("common") { "mapping_tracker.h", "metadata_tracker.cc", "metadata_tracker.h", + "process_track_translation_table.cc", + "process_track_translation_table.h", "process_tracker.cc", "process_tracker.h", "sched_event_state.h", @@ -87,6 +91,7 @@ source_set("common") { "../../util:build_id", "../../util:profiler_util", "../fuchsia:fuchsia_record", + "../perf:record", "../systrace:systrace_line", ] } @@ -116,6 +121,7 @@ source_set("unittests") { "deobfuscation_mapping_table_unittest.cc", "event_tracker_unittest.cc", "flow_tracker_unittest.cc", + "process_track_translation_table_unittest.cc", "process_tracker_unittest.cc", "slice_tracker_unittest.cc", "slice_translation_table_unittest.cc", diff --git a/src/trace_processor/importers/common/args_tracker.h b/src/trace_processor/importers/common/args_tracker.h index 964a6ad0e..e72b7c914 100644 --- a/src/trace_processor/importers/common/args_tracker.h +++ b/src/trace_processor/importers/common/args_tracker.h @@ -154,12 +154,31 @@ class ArgsTracker { context_->storage->mutable_surfaceflinger_transactions_table(), id); } + BoundInserter AddArgsTo(tables::ViewCaptureTable::Id id) { + return AddArgsTo(context_->storage->mutable_viewcapture_table(), id); + } + BoundInserter AddArgsTo(tables::WindowManagerShellTransitionsTable::Id id) { return AddArgsTo( context_->storage->mutable_window_manager_shell_transitions_table(), id); } + BoundInserter AddArgsTo(tables::AndroidKeyEventsTable::Id id) { + return AddArgsTo(context_->storage->mutable_android_key_events_table(), + id); + } + + BoundInserter AddArgsTo(tables::AndroidMotionEventsTable::Id id) { + return AddArgsTo(context_->storage->mutable_android_motion_events_table(), + id); + } + + BoundInserter AddArgsTo(tables::AndroidInputEventDispatchTable::Id id) { + return AddArgsTo( + context_->storage->mutable_android_input_event_dispatch_table(), id); + } + BoundInserter AddArgsTo(MetadataId id) { auto* table = context_->storage->mutable_metadata_table(); uint32_t row = *table->id().IndexOf(id); diff --git a/src/trace_processor/importers/common/async_track_set_tracker_unittest.cc b/src/trace_processor/importers/common/async_track_set_tracker_unittest.cc index b10aec821..173fd8921 100644 --- a/src/trace_processor/importers/common/async_track_set_tracker_unittest.cc +++ b/src/trace_processor/importers/common/async_track_set_tracker_unittest.cc @@ -18,6 +18,7 @@ #include "src/trace_processor/importers/common/args_tracker.h" #include "src/trace_processor/importers/common/global_args_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/types/trace_processor_context.h" #include "test/gtest_and_gmock.h" @@ -34,6 +35,8 @@ class AsyncTrackSetTrackerUnittest : public testing::Test { context_.args_tracker.reset(new ArgsTracker(&context_)); context_.track_tracker.reset(new TrackTracker(&context_)); context_.async_track_set_tracker.reset(new AsyncTrackSetTracker(&context_)); + context_.process_track_translation_table.reset( + new ProcessTrackTranslationTable(context_.storage.get())); storage_ = context_.storage.get(); tracker_ = context_.async_track_set_tracker.get(); diff --git a/src/trace_processor/importers/common/cpu_tracker.cc b/src/trace_processor/importers/common/cpu_tracker.cc new file mode 100644 index 000000000..b7039269b --- /dev/null +++ b/src/trace_processor/importers/common/cpu_tracker.cc @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/common/cpu_tracker.h" + +#include "perfetto/base/logging.h" +#include "perfetto/public/compiler.h" + +#include "src/trace_processor/importers/common/machine_tracker.h" + +namespace perfetto::trace_processor { + +CpuTracker::CpuTracker(TraceProcessorContext* context) : context_(context) { + // Preallocate ucpu of this machine for maintaining the relative order between + // ucpu and cpu. + auto machine_id = context_->machine_tracker->machine_id(); + if (machine_id.has_value()) + ucpu_offset_ = machine_id->value * kMaxCpusPerMachine; + + for (auto id = 0u; id < kMaxCpusPerMachine; id++) { + // Only populate the |machine_id| column. The |cpu| column is update only + // when the CPU is present. + tables::CpuTable::Row cpu_row; + cpu_row.machine_id = machine_id; + context_->storage->mutable_cpu_table()->Insert(cpu_row); + } +} + +CpuTracker::~CpuTracker() = default; + +tables::CpuTable::Id CpuTracker::SetCpuInfo(uint32_t cpu, + base::StringView processor, + uint32_t cluster_id) { + auto cpu_id = GetOrCreateCpu(cpu); + + auto cpu_row = context_->storage->mutable_cpu_table()->FindById(cpu_id); + PERFETTO_DCHECK(cpu_row.has_value()); + + if (!processor.empty()) { + auto string_id = context_->storage->InternString(processor); + cpu_row->set_processor(string_id); + } + cpu_row->set_cluster_id(cluster_id); + + return cpu_id; +} + +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/common/cpu_tracker.h b/src/trace_processor/importers/common/cpu_tracker.h new file mode 100644 index 000000000..31c0b61e9 --- /dev/null +++ b/src/trace_processor/importers/common/cpu_tracker.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_CPU_TRACKER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_CPU_TRACKER_H_ + +#include <bitset> + +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/metadata_tables_py.h" +#include "src/trace_processor/types/trace_processor_context.h" + +namespace perfetto::trace_processor { + +class TraceProcessorContext; + +class CpuTracker { + public: + // The CPU table Id serves as the 'ucpu' in sched_slice and related for + // joining with this table. To optimize for single-machine traces, this class + // assumes a maximum of |kMaxCpusPerMachine| CPUs per machine to maintain a + // relative order of |cpu| and |ucpu| by pre-allocating |kMaxCpusPerMachine| + // records in the CPU table. The mapping between |ucpu| and |cpu| becomes + // |cpu| = |ucpu| % |kMaxCpusPerMachine|. + static constexpr uint32_t kMaxCpusPerMachine = 4096; + + explicit CpuTracker(TraceProcessorContext*); + ~CpuTracker(); + + tables::CpuTable::Id GetOrCreateCpu(uint32_t cpu) { + // CPU core number is in the range of 0..kMaxCpusPerMachine-1. + PERFETTO_CHECK(cpu < kMaxCpusPerMachine); + auto ucpu = ucpu_offset_ + cpu; + if (PERFETTO_LIKELY(cpu_ids_[cpu])) + return tables::CpuTable::Id(ucpu); + + // Populate the optional |cpu| column. + context_->storage->mutable_cpu_table()->mutable_cpu()->Set(ucpu, cpu); + return tables::CpuTable::Id(ucpu); + } + + // Sets or updates the information for the specified CPU in the CpuTable. + tables::CpuTable::Id SetCpuInfo(uint32_t cpu, + base::StringView processor, + uint32_t cluster_id); + + private: + TraceProcessorContext* const context_; + + // Tracks the mapping of CPU number to CpuTable::Id of the current + // machine. + std::bitset<kMaxCpusPerMachine> cpu_ids_; + uint32_t ucpu_offset_ = 0; +}; + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_CPU_TRACKER_H_ diff --git a/src/trace_processor/importers/common/event_tracker.cc b/src/trace_processor/importers/common/event_tracker.cc index 43cf71051..a38d77624 100644 --- a/src/trace_processor/importers/common/event_tracker.cc +++ b/src/trace_processor/importers/common/event_tracker.cc @@ -16,20 +16,18 @@ #include "src/trace_processor/importers/common/event_tracker.h" -#include <math.h> +#include <cinttypes> +#include <cstdint> #include <optional> #include "perfetto/base/logging.h" -#include "perfetto/ext/base/utils.h" #include "src/trace_processor/importers/common/args_tracker.h" -#include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/storage/stats.h" +#include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/types/trace_processor_context.h" -#include "src/trace_processor/types/variadic.h" -namespace perfetto { -namespace trace_processor { +namespace perfetto::trace_processor { EventTracker::EventTracker(TraceProcessorContext* context) : context_(context) {} @@ -65,9 +63,7 @@ std::optional<CounterId> EventTracker::PushCounter(int64_t timestamp, max_timestamp_ = timestamp; auto* counter_values = context_->storage->mutable_counter_table(); - return counter_values - ->Insert({timestamp, track_id, value, {}, context_->machine_id()}) - .id; + return counter_values->Insert({timestamp, track_id, value, {}}).id; } std::optional<CounterId> EventTracker::PushCounter( @@ -106,5 +102,4 @@ void EventTracker::FlushPendingEvents() { pending_upid_resolution_counter_.clear(); } -} // namespace trace_processor -} // namespace perfetto +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/common/mapping_tracker.cc b/src/trace_processor/importers/common/mapping_tracker.cc index 0dec3e5ca..d0a2d8881 100644 --- a/src/trace_processor/importers/common/mapping_tracker.cc +++ b/src/trace_processor/importers/common/mapping_tracker.cc @@ -81,9 +81,6 @@ KernelMemoryMapping& MappingTracker::CreateKernelMemoryMapping( UserMemoryMapping& MappingTracker::CreateUserMemoryMapping( UniquePid upid, CreateMappingParams params) { - // TODO(carlscab): Guess build_id if not provided. Some tools like simpleperf - // add a mapping file_name ->build_id that we could use here - const AddressRange mapping_range = params.memory_range; std::unique_ptr<UserMemoryMapping> mapping( new UserMemoryMapping(context_, upid, std::move(params))); @@ -164,5 +161,15 @@ void MappingTracker::AddJitRange(UniquePid upid, }); } +VirtualMemoryMapping* MappingTracker::GetDummyMapping() { + if (!dummy_mapping_) { + CreateMappingParams params; + params.memory_range = + AddressRange::FromStartAndSize(0, std::numeric_limits<uint64_t>::max()); + dummy_mapping_ = &InternMemoryMapping(params); + } + return dummy_mapping_; +} + } // namespace trace_processor } // namespace perfetto diff --git a/src/trace_processor/importers/common/mapping_tracker.h b/src/trace_processor/importers/common/mapping_tracker.h index bc45befe8..08c767e0a 100644 --- a/src/trace_processor/importers/common/mapping_tracker.h +++ b/src/trace_processor/importers/common/mapping_tracker.h @@ -91,6 +91,10 @@ class MappingTracker { // Jitted ranges will only be applied to UserMemoryMappings void AddJitRange(UniquePid upid, AddressRange range, JitCache* jit_cache); + // Sometimes we just need a mapping and we are lacking trace data to create a + // proper one. Use this mapping in those cases. + VirtualMemoryMapping* GetDummyMapping(); + private: template <typename MappingImpl> MappingImpl& AddMapping(std::unique_ptr<MappingImpl> mapping); @@ -136,6 +140,8 @@ class MappingTracker { KernelMemoryMapping* kernel_ = nullptr; base::FlatHashMap<UniquePid, AddressRangeMap<JitCache*>> jit_caches_; + + VirtualMemoryMapping* dummy_mapping_; }; } // namespace trace_processor diff --git a/src/trace_processor/importers/common/process_track_translation_table.cc b/src/trace_processor/importers/common/process_track_translation_table.cc new file mode 100644 index 000000000..7cd542473 --- /dev/null +++ b/src/trace_processor/importers/common/process_track_translation_table.cc @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/common/process_track_translation_table.h" + +namespace perfetto::trace_processor { + +ProcessTrackTranslationTable::ProcessTrackTranslationTable( + TraceStorage* storage) + : storage_(storage) {} + +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/common/process_track_translation_table.h b/src/trace_processor/importers/common/process_track_translation_table.h new file mode 100644 index 000000000..30f5f56d5 --- /dev/null +++ b/src/trace_processor/importers/common/process_track_translation_table.h @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_PROCESS_TRACK_TRANSLATION_TABLE_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_PROCESS_TRACK_TRANSLATION_TABLE_H_ + +#include <cstdint> + +#include "perfetto/ext/base/flat_hash_map.h" +#include "perfetto/ext/base/string_view.h" +#include "src/trace_processor/storage/trace_storage.h" + +namespace perfetto::trace_processor { + +// Tracks and stores slice translation rules. It allows Trace Processor +// to for example deobfuscate slice names. +class ProcessTrackTranslationTable { + public: + ProcessTrackTranslationTable(TraceStorage* storage); + + // If the name is not mapped to anything, assumes that no translation is + // necessry, and returns the raw_name. + StringId TranslateName(StringId raw_name) const { + const auto* mapped_name = raw_to_deobfuscated_name_.Find(raw_name); + return mapped_name ? *mapped_name : raw_name; + } + + void AddNameTranslationRule(base::StringView raw, + base::StringView deobfuscated) { + const StringId raw_id = storage_->InternString(raw); + const StringId deobfuscated_id = storage_->InternString(deobfuscated); + raw_to_deobfuscated_name_[raw_id] = deobfuscated_id; + } + + private: + TraceStorage* storage_; + base::FlatHashMap<StringId, StringId> raw_to_deobfuscated_name_; +}; + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_PROCESS_TRACK_TRANSLATION_TABLE_H_ diff --git a/src/trace_processor/importers/common/process_track_translation_table_unittest.cc b/src/trace_processor/importers/common/process_track_translation_table_unittest.cc new file mode 100644 index 000000000..a947204ba --- /dev/null +++ b/src/trace_processor/importers/common/process_track_translation_table_unittest.cc @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/common/process_track_translation_table.h" +#include "test/gtest_and_gmock.h" + +namespace perfetto::trace_processor { +namespace { + +TEST(ProcessTrackTranslationTable, UnknownName) { + TraceStorage storage; + ProcessTrackTranslationTable table(&storage); + const StringId raw_name = storage.InternString("name1"); + EXPECT_EQ(raw_name, table.TranslateName(raw_name)); +} + +TEST(ProcessTrackTranslationTable, MappedName) { + TraceStorage storage; + ProcessTrackTranslationTable table(&storage); + table.AddNameTranslationRule("raw_name1", "mapped_name1"); + const StringId raw_name = storage.InternString("raw_name1"); + const StringId mapped_name = storage.InternString("mapped_name1"); + EXPECT_EQ(mapped_name, table.TranslateName(raw_name)); +} + +} // namespace +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/common/process_tracker.cc b/src/trace_processor/importers/common/process_tracker.cc index 3fc95fcbd..e29280174 100644 --- a/src/trace_processor/importers/common/process_tracker.cc +++ b/src/trace_processor/importers/common/process_tracker.cc @@ -356,6 +356,17 @@ UniquePid ProcessTracker::SetProcessMetadata(uint32_t pid, UniquePid upid = GetOrCreateProcess(pid); auto* process_table = context_->storage->mutable_process_table(); + // If we both know the previous and current parent pid and the two are not + // matching, we must have died and restarted: create a new process. + if (pupid) { + std::optional<UniquePid> prev_parent_upid = + process_table->parent_upid()[upid]; + if (prev_parent_upid && prev_parent_upid != pupid) { + upid = StartNewProcess(std::nullopt, ppid, pid, kNullStringId, + ThreadNamePriority::kOther); + } + } + StringId proc_name_id = context_->storage->InternString(name); process_table->mutable_name()->Set(upid, proc_name_id); process_table->mutable_cmdline()->Set( diff --git a/src/trace_processor/importers/common/sched_event_tracker.h b/src/trace_processor/importers/common/sched_event_tracker.h index 12aa4c02d..ff4f53709 100644 --- a/src/trace_processor/importers/common/sched_event_tracker.h +++ b/src/trace_processor/importers/common/sched_event_tracker.h @@ -19,6 +19,7 @@ #include "src/trace_processor/importers/common/event_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/types/destructible.h" #include "src/trace_processor/types/trace_processor_context.h" @@ -45,9 +46,10 @@ class SchedEventTracker : public Destructible { // just switched to. Set the duration to -1, to indicate that the event is // not finished. Duration will be updated later after event finish. auto* sched = context_->storage->mutable_sched_slice_table(); - auto row_and_id = - sched->Insert({ts, /* duration */ -1, cpu, next_utid, kNullStringId, - next_prio, context_->machine_id()}); + // Get the unique CPU Id over all machines from the CPU table. + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); + auto row_and_id = sched->Insert( + {ts, /* duration */ -1, next_utid, kNullStringId, next_prio, ucpu}); SchedId sched_id = row_and_id.id; return *sched->id().IndexOf(sched_id); } diff --git a/src/trace_processor/importers/common/thread_state_tracker.cc b/src/trace_processor/importers/common/thread_state_tracker.cc index 2d98ffb1a..a0d98ce74 100644 --- a/src/trace_processor/importers/common/thread_state_tracker.cc +++ b/src/trace_processor/importers/common/thread_state_tracker.cc @@ -19,6 +19,7 @@ #include <cstdint> #include <optional> +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/process_tracker.h" namespace perfetto { @@ -137,12 +138,12 @@ void ThreadStateTracker::AddOpenState(int64_t ts, // Insert row with unfinished state tables::ThreadStateTable::Row row; row.ts = ts; - row.cpu = cpu; row.waker_utid = waker_utid; row.dur = -1; row.utid = utid; row.state = state; - row.machine_id = context_->machine_id(); + if (cpu) + row.ucpu = context_->cpu_tracker->GetOrCreateCpu(*cpu); if (common_flags.has_value()) { row.irq_context = CommonFlagsToIrqContext(*common_flags); } diff --git a/src/trace_processor/importers/common/thread_state_tracker_unittest.cc b/src/trace_processor/importers/common/thread_state_tracker_unittest.cc index 153b02ba8..12381413e 100644 --- a/src/trace_processor/importers/common/thread_state_tracker_unittest.cc +++ b/src/trace_processor/importers/common/thread_state_tracker_unittest.cc @@ -19,7 +19,9 @@ #include <algorithm> #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/global_args_tracker.h" +#include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/types/trace_processor_context.h" #include "test/gtest_and_gmock.h" @@ -45,6 +47,8 @@ class ThreadStateTrackerUnittest : public testing::Test { context_.process_tracker.reset(new ProcessTracker(&context_)); context_.global_args_tracker.reset( new GlobalArgsTracker(context_.storage.get())); + context_.machine_tracker.reset(new MachineTracker(&context_, 0)); + context_.cpu_tracker.reset(new CpuTracker(&context_)); context_.args_tracker.reset(new ArgsTracker(&context_)); tracker_.reset(new ThreadStateTracker(&context_)); } @@ -72,12 +76,12 @@ class ThreadStateTrackerUnittest : public testing::Test { ASSERT_EQ(it.utid(), utid); if (state == kRunning) { if (cpu.has_value()) { - ASSERT_EQ(it.cpu(), cpu); + ASSERT_EQ(it.ucpu().value().value, cpu); } else { - ASSERT_EQ(it.cpu(), CPU_A); + ASSERT_EQ(it.ucpu().value().value, CPU_A); } } else { - ASSERT_EQ(it.cpu(), std::nullopt); + ASSERT_EQ(it.ucpu(), std::nullopt); } ASSERT_STREQ(context_.storage->GetString(it.state()).c_str(), state); ASSERT_EQ(it.io_wait(), io_wait); diff --git a/src/trace_processor/importers/common/trace_parser.h b/src/trace_processor/importers/common/trace_parser.h index bf90c95fb..8e41a7dba 100644 --- a/src/trace_processor/importers/common/trace_parser.h +++ b/src/trace_processor/importers/common/trace_parser.h @@ -20,8 +20,10 @@ #include <stdint.h> #include <string> -namespace perfetto { -namespace trace_processor { +namespace perfetto::trace_processor { +namespace perf_importer { +struct Record; +} class PacketSequenceStateGeneration; class TraceBlobView; @@ -59,10 +61,9 @@ class FuchsiaRecordParser { class PerfRecordParser { public: virtual ~PerfRecordParser(); - virtual void ParsePerfRecord(int64_t, TraceBlobView) = 0; + virtual void ParsePerfRecord(int64_t, perf_importer::Record) = 0; }; -} // namespace trace_processor -} // namespace perfetto +} // namespace perfetto::trace_processor #endif // SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_TRACE_PARSER_H_ diff --git a/src/trace_processor/importers/common/track_tracker.cc b/src/trace_processor/importers/common/track_tracker.cc index 502639152..a2ba1f8d4 100644 --- a/src/trace_processor/importers/common/track_tracker.cc +++ b/src/trace_processor/importers/common/track_tracker.cc @@ -19,7 +19,10 @@ #include <optional> #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/profiler_tables_py.h" +#include "src/trace_processor/types/trace_processor_context.h" namespace perfetto { namespace trace_processor { @@ -140,7 +143,7 @@ TrackId TrackTracker::InternGpuWorkPeriodTrack( } TrackId TrackTracker::InternLegacyChromeAsyncTrack( - StringId name, + StringId raw_name, uint32_t upid, int64_t trace_id, bool trace_id_is_process_scoped, @@ -151,6 +154,8 @@ TrackId TrackTracker::InternLegacyChromeAsyncTrack( tuple.trace_id = trace_id; tuple.source_scope = source_scope; + const StringId name = + context_->process_track_translation_table->TranslateName(raw_name); auto it = chrome_tracks_.find(tuple); if (it != chrome_tracks_.end()) { if (name != kNullStringId) { @@ -194,9 +199,11 @@ TrackId TrackTracker::CreateGlobalAsyncTrack(StringId name, StringId source) { return id; } -TrackId TrackTracker::CreateProcessAsyncTrack(StringId name, +TrackId TrackTracker::CreateProcessAsyncTrack(StringId raw_name, UniquePid upid, StringId source) { + const StringId name = + context_->process_track_translation_table->TranslateName(raw_name); tables::ProcessTrackTable::Row row(name); row.upid = upid; row.machine_id = context_->machine_id(); @@ -318,10 +325,12 @@ TrackId TrackTracker::InternThreadCounterTrack(StringId name, UniqueTid utid) { return track; } -TrackId TrackTracker::InternProcessCounterTrack(StringId name, +TrackId TrackTracker::InternProcessCounterTrack(StringId raw_name, UniquePid upid, StringId unit, StringId description) { + const StringId name = + context_->process_track_translation_table->TranslateName(raw_name); auto it = upid_counter_tracks_.find(std::make_pair(name, upid)); if (it != upid_counter_tracks_.end()) { return it->second; @@ -447,10 +456,11 @@ TrackId TrackTracker::CreateGpuCounterTrack(StringId name, return context_->storage->mutable_gpu_counter_track_table()->Insert(row).id; } -TrackId TrackTracker::CreatePerfCounterTrack(StringId name, - uint32_t perf_session_id, - uint32_t cpu, - bool is_timebase) { +TrackId TrackTracker::CreatePerfCounterTrack( + StringId name, + tables::PerfSessionTable::Id perf_session_id, + uint32_t cpu, + bool is_timebase) { tables::PerfCounterTrackTable::Row row(name); row.perf_session_id = perf_session_id; row.cpu = cpu; diff --git a/src/trace_processor/importers/common/track_tracker.h b/src/trace_processor/importers/common/track_tracker.h index 0fec299ab..c0662049b 100644 --- a/src/trace_processor/importers/common/track_tracker.h +++ b/src/trace_processor/importers/common/track_tracker.h @@ -18,8 +18,10 @@ #define SRC_TRACE_PROCESSOR_IMPORTERS_COMMON_TRACK_TRACKER_H_ #include <optional> + #include "src/trace_processor/importers/common/args_tracker.h" #include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/profiler_tables_py.h" #include "src/trace_processor/types/trace_processor_context.h" namespace perfetto { @@ -145,7 +147,7 @@ class TrackTracker { // Creates a counter track for values within perf samples. // The tracks themselves are managed by PerfSampleTracker. TrackId CreatePerfCounterTrack(StringId name, - uint32_t perf_session_id, + tables::PerfSessionTable::Id perf_session_id, uint32_t cpu, bool is_timebase); diff --git a/src/trace_processor/importers/ftrace/ftrace_descriptors.cc b/src/trace_processor/importers/ftrace/ftrace_descriptors.cc index 64ae4d574..91bfb5e31 100644 --- a/src/trace_processor/importers/ftrace/ftrace_descriptors.cc +++ b/src/trace_processor/importers/ftrace/ftrace_descriptors.cc @@ -24,7 +24,7 @@ namespace perfetto { namespace trace_processor { namespace { -std::array<FtraceMessageDescriptor, 508> descriptors{{ +std::array<FtraceMessageDescriptor, 510> descriptors{{ {nullptr, 0, {}}, {nullptr, 0, {}}, {nullptr, 0, {}}, @@ -5619,6 +5619,24 @@ std::array<FtraceMessageDescriptor, 508> descriptors{{ {"k_i", ProtoSchemaType::kInt32}, }, }, + { + "dcvsh_freq", + 2, + { + {}, + {"cpu", ProtoSchemaType::kUint64}, + {"freq", ProtoSchemaType::kUint64}, + }, + }, + { + "kgsl_gpu_frequency", + 2, + { + {}, + {"gpu_freq", ProtoSchemaType::kUint32}, + {"gpu_id", ProtoSchemaType::kUint32}, + }, + }, }}; } // namespace diff --git a/src/trace_processor/importers/ftrace/ftrace_parser.cc b/src/trace_processor/importers/ftrace/ftrace_parser.cc index 406f30199..75bfd00eb 100644 --- a/src/trace_processor/importers/ftrace/ftrace_parser.cc +++ b/src/trace_processor/importers/ftrace/ftrace_parser.cc @@ -24,6 +24,7 @@ #include "perfetto/trace_processor/basic_types.h" #include "src/trace_processor/importers/common/args_tracker.h" #include "src/trace_processor/importers/common/async_track_set_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/metadata_tracker.h" #include "src/trace_processor/importers/common/parser_types.h" #include "src/trace_processor/importers/common/process_tracker.h" @@ -49,6 +50,7 @@ #include "protos/perfetto/trace/ftrace/cma.pbzero.h" #include "protos/perfetto/trace/ftrace/cpuhp.pbzero.h" #include "protos/perfetto/trace/ftrace/cros_ec.pbzero.h" +#include "protos/perfetto/trace/ftrace/dcvsh.pbzero.h" #include "protos/perfetto/trace/ftrace/dmabuf_heap.pbzero.h" #include "protos/perfetto/trace/ftrace/dpu.pbzero.h" #include "protos/perfetto/trace/ftrace/fastrpc.pbzero.h" @@ -63,6 +65,7 @@ #include "protos/perfetto/trace/ftrace/i2c.pbzero.h" #include "protos/perfetto/trace/ftrace/ion.pbzero.h" #include "protos/perfetto/trace/ftrace/irq.pbzero.h" +#include "protos/perfetto/trace/ftrace/kgsl.pbzero.h" #include "protos/perfetto/trace/ftrace/kmem.pbzero.h" #include "protos/perfetto/trace/ftrace/lwis.pbzero.h" #include "protos/perfetto/trace/ftrace/mali.pbzero.h" @@ -317,6 +320,8 @@ FtraceParser::FtraceParser(TraceProcessorContext* context) sched_waking_name_id_(context->storage->InternString("sched_waking")), cpu_id_(context->storage->InternString("cpu")), cpu_freq_name_id_(context->storage->InternString("cpufreq")), + cpu_freq_throttle_name_id_( + context->storage->InternString("cpufreq_throttle")), gpu_freq_name_id_(context->storage->InternString("gpufreq")), cpu_idle_name_id_(context->storage->InternString("cpuidle")), suspend_resume_name_id_( @@ -761,10 +766,18 @@ base::Status FtraceParser::ParseFtraceEvent(uint32_t cpu, ParseCpuFreq(ts, fld_bytes); break; } + case FtraceEvent::kDcvshFreqFieldNumber: { + ParseCpuFreqThrottle(ts, fld_bytes); + break; + } case FtraceEvent::kGpuFrequencyFieldNumber: { ParseGpuFreq(ts, fld_bytes); break; } + case FtraceEvent::kKgslGpuFrequencyFieldNumber: { + ParseKgslGpuFreq(ts, fld_bytes); + break; + } case FtraceEvent::kCpuIdleFieldNumber: { ParseCpuIdle(ts, fld_bytes); break; @@ -951,8 +964,7 @@ base::Status FtraceParser::ParseFtraceEvent(uint32_t cpu, break; } case FtraceEvent::kThermalExynosAcpmHighOverheadFieldNumber: { - thermal_tracker_.ParseThermalExynosAcpmHighOverhead( - ts, fld_bytes); + thermal_tracker_.ParseThermalExynosAcpmHighOverhead(ts, fld_bytes); break; } case FtraceEvent::kCdevUpdateFieldNumber: { @@ -1362,10 +1374,10 @@ void FtraceParser::ParseGenericFtrace(int64_t ts, protos::pbzero::GenericFtraceEvent::Decoder evt(blob.data, blob.size); StringId event_id = context_->storage->InternString(evt.event_name()); UniqueTid utid = context_->process_tracker->GetOrCreateThread(tid); - RawId id = - context_->storage->mutable_ftrace_event_table() - ->Insert({ts, event_id, cpu, utid, {}, {}, context_->machine_id()}) - .id; + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); + RawId id = context_->storage->mutable_ftrace_event_table() + ->Insert({ts, event_id, utid, {}, {}, ucpu}) + .id; auto inserter = context_->args_tracker->AddArgsTo(id); for (auto it = evt.field(); it; ++it) { @@ -1404,15 +1416,12 @@ void FtraceParser::ParseTypedFtraceToRaw( FtraceMessageDescriptor* m = GetMessageDescriptorForId(ftrace_id); const auto& message_strings = ftrace_message_strings_[ftrace_id]; UniqueTid utid = context_->process_tracker->GetOrCreateThread(tid); - RawId id = context_->storage->mutable_ftrace_event_table() - ->Insert({timestamp, - message_strings.message_name_id, - cpu, - utid, - {}, - {}, - context_->machine_id()}) - .id; + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); + RawId id = + context_->storage->mutable_ftrace_event_table() + ->Insert( + {timestamp, message_strings.message_name_id, utid, {}, {}, ucpu}) + .id; auto inserter = context_->args_tracker->AddArgsTo(id); for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) { @@ -1542,6 +1551,16 @@ void FtraceParser::ParseCpuFreq(int64_t timestamp, ConstBytes blob) { context_->event_tracker->PushCounter(timestamp, new_freq, track); } +void FtraceParser::ParseCpuFreqThrottle(int64_t timestamp, ConstBytes blob) { + protos::pbzero::DcvshFreqFtraceEvent::Decoder freq(blob.data, blob.size); + uint32_t cpu = static_cast<uint32_t>(freq.cpu()); + // Source data is frequency / 1000, so we correct that here: + double new_freq = static_cast<double>(freq.freq()) * 1000.0; + TrackId track = context_->track_tracker->InternCpuCounterTrack( + cpu_freq_throttle_name_id_, cpu); + context_->event_tracker->PushCounter(timestamp, new_freq, track); +} + void FtraceParser::ParseGpuFreq(int64_t timestamp, ConstBytes blob) { protos::pbzero::GpuFrequencyFtraceEvent::Decoder freq(blob.data, blob.size); uint32_t gpu = freq.gpu_id(); @@ -1551,6 +1570,17 @@ void FtraceParser::ParseGpuFreq(int64_t timestamp, ConstBytes blob) { context_->event_tracker->PushCounter(timestamp, new_freq, track); } +void FtraceParser::ParseKgslGpuFreq(int64_t timestamp, ConstBytes blob) { + protos::pbzero::KgslGpuFrequencyFtraceEvent::Decoder freq(blob.data, + blob.size); + uint32_t gpu = freq.gpu_id(); + // Source data is frequency / 1000, so we correct that here: + double new_freq = static_cast<double>(freq.gpu_freq()) * 1000.0; + TrackId track = + context_->track_tracker->InternGpuCounterTrack(gpu_freq_name_id_, gpu); + context_->event_tracker->PushCounter(timestamp, new_freq, track); +} + void FtraceParser::ParseCpuIdle(int64_t timestamp, ConstBytes blob) { protos::pbzero::CpuIdleFtraceEvent::Decoder idle(blob.data, blob.size); uint32_t cpu = idle.cpu_id(); diff --git a/src/trace_processor/importers/ftrace/ftrace_parser.h b/src/trace_processor/importers/ftrace/ftrace_parser.h index d8bafde94..2520572de 100644 --- a/src/trace_processor/importers/ftrace/ftrace_parser.h +++ b/src/trace_processor/importers/ftrace/ftrace_parser.h @@ -74,7 +74,9 @@ class FtraceParser { void ParseSchedWaking(int64_t timestamp, uint32_t pid, protozero::ConstBytes); void ParseSchedProcessFree(int64_t timestamp, protozero::ConstBytes); void ParseCpuFreq(int64_t timestamp, protozero::ConstBytes); + void ParseCpuFreqThrottle(int64_t timestamp, protozero::ConstBytes); void ParseGpuFreq(int64_t timestamp, protozero::ConstBytes); + void ParseKgslGpuFreq(int64_t timestamp, protozero::ConstBytes); void ParseCpuIdle(int64_t timestamp, protozero::ConstBytes); void ParsePrint(int64_t timestamp, uint32_t pid, protozero::ConstBytes); void ParseZero(int64_t timestamp, uint32_t pid, protozero::ConstBytes); @@ -319,6 +321,7 @@ class FtraceParser { const StringId sched_waking_name_id_; const StringId cpu_id_; const StringId cpu_freq_name_id_; + const StringId cpu_freq_throttle_name_id_; const StringId gpu_freq_name_id_; const StringId cpu_idle_name_id_; const StringId suspend_resume_name_id_; diff --git a/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker.cc b/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker.cc index a7b1ced40..feb616c31 100644 --- a/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker.cc +++ b/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker.cc @@ -238,10 +238,9 @@ void FtraceSchedEventTracker::PushSchedWakingCompact(uint32_t cpu, tables::FtraceEventTable::Row row; row.ts = ts; row.name = sched_waking_id_; - row.cpu = cpu; row.utid = curr_utid; row.common_flags = common_flags; - row.machine_id = context_->machine_id(); + row.ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); // Add an entry to the raw table. RawId id = context_->storage->mutable_ftrace_event_table()->Insert(row).id; @@ -280,14 +279,9 @@ void FtraceSchedEventTracker::AddRawSchedSwitchEvent(uint32_t cpu, if (PERFETTO_LIKELY(context_->config.ingest_ftrace_in_raw_table)) { // Push the raw event - this is done as the raw ftrace event codepath does // not insert sched_switch. + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); RawId id = context_->storage->mutable_ftrace_event_table() - ->Insert({ts, - sched_switch_id_, - cpu, - prev_utid, - {}, - {}, - context_->machine_id()}) + ->Insert({ts, sched_switch_id_, prev_utid, {}, {}, ucpu}) .id; // Note: this ordering is important. The events should be pushed in the same diff --git a/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker_unittest.cc b/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker_unittest.cc index a085078cd..8becd2ed9 100644 --- a/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker_unittest.cc +++ b/src/trace_processor/importers/ftrace/ftrace_sched_event_tracker_unittest.cc @@ -18,9 +18,11 @@ #include "perfetto/base/logging.h" #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/event_tracker.h" -#include "src/trace_processor/importers/common/sched_event_tracker.h" +#include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/process_tracker.h" +#include "src/trace_processor/importers/common/sched_event_tracker.h" #include "test/gtest_and_gmock.h" namespace perfetto { @@ -40,6 +42,8 @@ class SchedEventTrackerTest : public ::testing::Test { context.args_tracker.reset(new ArgsTracker(&context)); context.event_tracker.reset(new EventTracker(&context)); context.process_tracker.reset(new ProcessTracker(&context)); + context.machine_tracker.reset(new MachineTracker(&context, 0)); + context.cpu_tracker.reset(new CpuTracker(&context)); context.sched_event_tracker.reset(new SchedEventTracker(&context)); sched_tracker = FtraceSchedEventTracker::GetOrCreate(&context); } diff --git a/src/trace_processor/importers/ftrace/ftrace_tokenizer.cc b/src/trace_processor/importers/ftrace/ftrace_tokenizer.cc index 02209863b..1e61a465f 100644 --- a/src/trace_processor/importers/ftrace/ftrace_tokenizer.cc +++ b/src/trace_processor/importers/ftrace/ftrace_tokenizer.cc @@ -271,7 +271,8 @@ void FtraceTokenizer::TokenizeFtraceEvent( } else if (PERFETTO_UNLIKELY(event_id == protos::pbzero::FtraceEvent:: kThermalExynosAcpmBulkFieldNumber)) { - TokenizeFtraceThermalExynosAcpmBulk(cpu, std::move(event), std::move(state)); + TokenizeFtraceThermalExynosAcpmBulk(cpu, std::move(event), + std::move(state)); return; } @@ -475,7 +476,8 @@ void FtraceTokenizer::TokenizeFtraceGpuWorkPeriod( } void FtraceTokenizer::TokenizeFtraceThermalExynosAcpmBulk( - uint32_t cpu, TraceBlobView event, + uint32_t cpu, + TraceBlobView event, RefPtr<PacketSequenceStateGeneration> state) { // Special handling of valid thermal_exynos_acpm_bulk tracepoint events which // contains the right timestamp value nested inside the event data. diff --git a/src/trace_processor/importers/ftrace/ftrace_tokenizer.h b/src/trace_processor/importers/ftrace/ftrace_tokenizer.h index e0b4f7c95..6aff47da1 100644 --- a/src/trace_processor/importers/ftrace/ftrace_tokenizer.h +++ b/src/trace_processor/importers/ftrace/ftrace_tokenizer.h @@ -64,9 +64,10 @@ class FtraceTokenizer { void TokenizeFtraceGpuWorkPeriod(uint32_t cpu, TraceBlobView event, RefPtr<PacketSequenceStateGeneration> state); - void TokenizeFtraceThermalExynosAcpmBulk(uint32_t cpu, - TraceBlobView event, - RefPtr<PacketSequenceStateGeneration> state); + void TokenizeFtraceThermalExynosAcpmBulk( + uint32_t cpu, + TraceBlobView event, + RefPtr<PacketSequenceStateGeneration> state); void DlogWithLimit(const base::Status& status) { static std::atomic<uint32_t> dlog_count(0); diff --git a/src/trace_processor/importers/ftrace/thermal_tracker.cc b/src/trace_processor/importers/ftrace/thermal_tracker.cc index 4e1e367f5..f2fe7bb16 100644 --- a/src/trace_processor/importers/ftrace/thermal_tracker.cc +++ b/src/trace_processor/importers/ftrace/thermal_tracker.cc @@ -14,12 +14,12 @@ * limitations under the License. */ +#include "src/trace_processor/importers/ftrace/thermal_tracker.h" #include "perfetto/ext/base/string_utils.h" #include "protos/perfetto/trace/ftrace/thermal.pbzero.h" #include "protos/perfetto/trace/ftrace/thermal_exynos.pbzero.h" #include "src/trace_processor/importers/common/event_tracker.h" #include "src/trace_processor/importers/common/track_tracker.h" -#include "src/trace_processor/importers/ftrace/thermal_tracker.h" namespace perfetto { namespace trace_processor { @@ -84,7 +84,8 @@ void ThermalTracker::ParseThermalExynosAcpmBulk(protozero::ConstBytes blob) { } void ThermalTracker::ParseThermalExynosAcpmHighOverhead( - int64_t timestamp, protozero::ConstBytes blob) { + int64_t timestamp, + protozero::ConstBytes blob) { protos::pbzero::ThermalExynosAcpmHighOverheadFtraceEvent::Decoder event(blob); auto tz_id = static_cast<size_t>(event.tz_id()); if (tz_id >= kAcpmThermalZones) { @@ -100,7 +101,8 @@ void ThermalTracker::ParseThermalExynosAcpmHighOverhead( static_cast<double>(event.cdev_state())); } -void ThermalTracker::PushCounter(int64_t timestamp, StringId counter_id, +void ThermalTracker::PushCounter(int64_t timestamp, + StringId counter_id, double value) { TrackId track = context_->track_tracker->InternGlobalCounterTrack( TrackTracker::Group::kThermals, counter_id); diff --git a/src/trace_processor/importers/fuchsia/fuchsia_parser_unittest.cc b/src/trace_processor/importers/fuchsia/fuchsia_parser_unittest.cc index ca02bdcb0..740640873 100644 --- a/src/trace_processor/importers/fuchsia/fuchsia_parser_unittest.cc +++ b/src/trace_processor/importers/fuchsia/fuchsia_parser_unittest.cc @@ -14,25 +14,29 @@ * limitations under the License. */ -#include "src/trace_processor/importers/common/trace_parser.h" #include "src/trace_processor/importers/fuchsia/fuchsia_trace_parser.h" -#include "src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.h" #include "perfetto/base/logging.h" #include "perfetto/ext/base/string_view.h" #include "perfetto/protozero/scattered_heap_buffer.h" #include "perfetto/trace_processor/trace_blob.h" + #include "src/trace_processor/importers/common/args_tracker.h" #include "src/trace_processor/importers/common/args_translation_table.h" #include "src/trace_processor/importers/common/clock_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/event_tracker.h" #include "src/trace_processor/importers/common/flow_tracker.h" +#include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/metadata_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/slice_tracker.h" #include "src/trace_processor/importers/common/stack_profile_tracker.h" +#include "src/trace_processor/importers/common/trace_parser.h" #include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/importers/ftrace/ftrace_sched_event_tracker.h" +#include "src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.h" #include "src/trace_processor/importers/proto/additional_modules.h" #include "src/trace_processor/importers/proto/default_modules.h" #include "src/trace_processor/importers/proto/proto_trace_parser_impl.h" @@ -242,12 +246,16 @@ class FuchsiaTraceParserTest : public ::testing::Test { context_.args_translation_table.reset(new ArgsTranslationTable(storage_)); context_.metadata_tracker.reset( new MetadataTracker(context_.storage.get())); + context_.machine_tracker.reset(new MachineTracker(&context_, 0)); + context_.cpu_tracker.reset(new CpuTracker(&context_)); event_ = new MockEventTracker(&context_); context_.event_tracker.reset(event_); sched_ = new MockSchedEventTracker(&context_); context_.ftrace_sched_tracker.reset(sched_); process_ = new NiceMock<MockProcessTracker>(&context_); context_.process_tracker.reset(process_); + context_.process_track_translation_table.reset( + new ProcessTrackTranslationTable(storage_)); slice_ = new NiceMock<MockSliceTracker>(&context_); context_.slice_tracker.reset(slice_); context_.slice_translation_table.reset(new SliceTranslationTable(storage_)); diff --git a/src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.cc b/src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.cc index 0cbd4f787..a668c42d3 100644 --- a/src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.cc +++ b/src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.cc @@ -22,6 +22,7 @@ #include "perfetto/base/logging.h" #include "perfetto/ext/base/string_view.h" #include "perfetto/trace_processor/trace_blob.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/slice_tracker.h" #include "src/trace_processor/importers/fuchsia/fuchsia_record.h" @@ -257,7 +258,7 @@ void FuchsiaTraceTokenizer::SwitchFrom(Thread* thread, // state. tables::ThreadStateTable::Row state_row; state_row.ts = ts; - state_row.cpu = cpu; + state_row.ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); state_row.dur = -1; state_row.state = state; state_row.utid = utid; @@ -287,10 +288,11 @@ void FuchsiaTraceTokenizer::SwitchTo(Thread* thread, thread->last_state_row.reset(); } + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); // Open a new slice record for this thread. tables::SchedSliceTable::Row slice_row; slice_row.ts = ts; - slice_row.cpu = cpu; + slice_row.ucpu = ucpu; slice_row.dur = -1; slice_row.utid = utid; slice_row.priority = weight; @@ -301,7 +303,7 @@ void FuchsiaTraceTokenizer::SwitchTo(Thread* thread, // Open a new state record for this thread. tables::ThreadStateTable::Row state_row; state_row.ts = ts; - state_row.cpu = cpu; + state_row.ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); state_row.dur = -1; state_row.state = running_string_id_; state_row.utid = utid; @@ -331,7 +333,7 @@ void FuchsiaTraceTokenizer::Wake(Thread* thread, int64_t ts, uint32_t cpu) { // Open a new state record for this thread. tables::ThreadStateTable::Row state_row; state_row.ts = ts; - state_row.cpu = cpu; + state_row.ucpu = context_->cpu_tracker->GetOrCreateCpu(cpu); state_row.dur = -1; state_row.state = waking_string_id_; state_row.utid = utid; diff --git a/src/trace_processor/importers/perf/BUILD.gn b/src/trace_processor/importers/perf/BUILD.gn index 376f9b420..da4153a6d 100644 --- a/src/trace_processor/importers/perf/BUILD.gn +++ b/src/trace_processor/importers/perf/BUILD.gn @@ -16,6 +16,8 @@ import("../../../../gn/test.gni") source_set("record") { sources = [ + "perf_counter.cc", + "perf_counter.h", "perf_event.h", "perf_event_attr.cc", "perf_event_attr.h", @@ -26,38 +28,64 @@ source_set("record") { ] deps = [ "../../../../gn:default_deps", + "../../../../include/perfetto/ext/base:base", + "../../../../include/perfetto/trace_processor:trace_processor", "../../../../protos/perfetto/trace/profiling:zero", - "../../sorter", "../../storage", "../../tables:tables_python", "../../types", - "../common", + "../../util:build_id", "../common:parser_types", ] } + +source_set("tracker") { + sources = [ + "dso_tracker.cc", + "dso_tracker.h", + ] + deps = [ + "../../../../gn:default_deps", + "../../../../include/perfetto/ext/base:base", + "../../../../protos/third_party/simpleperf:zero", + "../..//storage:storage", + "../..//tables:tables", + "../..//types:types", + "../common:common", + ] +} + source_set("perf") { sources = [ - "perf_data_parser.cc", - "perf_data_parser.h", - "perf_data_reader.cc", - "perf_data_reader.h", + "attrs_section_reader.cc", + "attrs_section_reader.h", + "features.cc", + "features.h", + "mmap_record.cc", + "mmap_record.h", "perf_data_tokenizer.cc", "perf_data_tokenizer.h", - "perf_data_tracker.cc", - "perf_data_tracker.h", "perf_file.h", - "reader.h", + "record_parser.cc", + "record_parser.h", + "sample.cc", + "sample.h", ] public_deps = [ ":record" ] deps = [ + ":tracker", "../../../../gn:default_deps", + "../../../../protos/perfetto/trace:zero", "../../../../protos/perfetto/trace/profiling:zero", + "../../../../protos/third_party/simpleperf:zero", "../../sorter", "../../storage", "../../tables:tables_python", "../../types", - "../common", - "../common:parser_types", + "../../util:build_id", + "../../util:file_buffer", + "../../util:util", + "../common:common", "../proto:minimal", ] } @@ -65,8 +93,6 @@ source_set("perf") { perfetto_unittest_source_set("unittests") { testonly = true sources = [ - "perf_data_reader_unittest.cc", - "perf_data_tracker_unittest.cc", "perf_session_unittest.cc", "reader_unittest.cc", ] @@ -76,6 +102,8 @@ perfetto_unittest_source_set("unittests") { "../../../../gn:gtest_and_gmock", "../../../../protos/perfetto/trace/profiling:zero", "../../../base", - "../../importers/common", + "../../storage", + "../../types", + "../common", ] } diff --git a/src/trace_processor/importers/perf/attrs_section_reader.cc b/src/trace_processor/importers/perf/attrs_section_reader.cc new file mode 100644 index 000000000..f19ea3efc --- /dev/null +++ b/src/trace_processor/importers/perf/attrs_section_reader.cc @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/perf/attrs_section_reader.h" + +#include <cinttypes> + +#include "perfetto/base/logging.h" +#include "perfetto/base/status.h" +#include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/importers/perf/perf_file.h" + +namespace perfetto::trace_processor::perf_importer { + +// static +base::StatusOr<AttrsSectionReader> AttrsSectionReader::Create( + const PerfFile::Header& header, + TraceBlobView section) { + PERFETTO_CHECK(section.size() == header.attrs.size); + + if (header.attr_size == 0) { + return base::ErrStatus("Invalid attr_size (0) in perf file header."); + } + + if (header.attrs.size % header.attr_size != 0) { + return base::ErrStatus("Invalid attrs section size %" PRIu64 + " for attr_size %" PRIu64 " in perf file header.", + header.attrs.size, header.attr_size); + } + + const size_t num_attr = header.attrs.size / header.attr_size; + + // Each entry is a perf_event_attr followed by a Section, but the size of + // the perf_event_attr struct written in the file might not be the same as + // sizeof(perf_event_attr) as this struct might grow over time (can be + // bigger or smaller). + static constexpr size_t kSectionSize = sizeof(PerfFile::Section); + if (header.attr_size < kSectionSize) { + return base::ErrStatus( + "Invalid attr_size in file header. Expected at least %zu, found " + "%" PRIu64, + kSectionSize, header.attr_size); + } + const size_t attr_size = header.attr_size - kSectionSize; + + return AttrsSectionReader(std::move(section), num_attr, attr_size); +} + +base::Status AttrsSectionReader::ReadNext(PerfFile::AttrsEntry& entry) { + PERFETTO_CHECK(reader_.ReadPerfEventAttr(entry.attr, attr_size_)); + + if (entry.attr.size != attr_size_) { + return base::ErrStatus( + "Invalid attr.size. Expected %zu, but found %" PRIu32, attr_size_, + entry.attr.size); + } + + PERFETTO_CHECK(reader_.Read(entry.ids)); + --num_attr_; + return base::OkStatus(); +} + +} // namespace perfetto::trace_processor::perf_importer diff --git a/src/trace_processor/importers/perf/attrs_section_reader.h b/src/trace_processor/importers/perf/attrs_section_reader.h new file mode 100644 index 000000000..a7eaaa3e7 --- /dev/null +++ b/src/trace_processor/importers/perf/attrs_section_reader.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_ATTRS_SECTION_READER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_ATTRS_SECTION_READER_H_ + +#include "perfetto/ext/base/status_or.h" +#include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/importers/perf/perf_file.h" +#include "src/trace_processor/importers/perf/reader.h" + +namespace perfetto::trace_processor::perf_importer { + +// Helper to read the attrs section of a perf file. Provides an iterator like +// interface over the perf_event_attr entries. +class AttrsSectionReader { + public: + // Creates a new iterator. + // `attrs_section` data contained in the attrs section of the perf file. + static base::StatusOr<AttrsSectionReader> Create( + const PerfFile::Header& header, + TraceBlobView attrs_section); + + // Returns true while there are available entries to read via `ReadNext`. + bool CanReadNext() const { return num_attr_ != 0; } + + // Reads the next entry. Can onlybe called if `HasMore` returns true. + base::Status ReadNext(PerfFile::AttrsEntry& entry); + + private: + AttrsSectionReader(TraceBlobView section, size_t num_attr, size_t attr_size) + : reader_(std::move(section)), + num_attr_(num_attr), + attr_size_(attr_size) {} + + Reader reader_; + size_t num_attr_; + const size_t attr_size_; +}; + +} // namespace perfetto::trace_processor::perf_importer + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_ATTRS_SECTION_READER_H_ diff --git a/src/trace_processor/importers/perf/dso_tracker.cc b/src/trace_processor/importers/perf/dso_tracker.cc new file mode 100644 index 000000000..3ff3cefff --- /dev/null +++ b/src/trace_processor/importers/perf/dso_tracker.cc @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/perf/dso_tracker.h" + +#include <cstdint> + +#include "perfetto/base/status.h" +#include "perfetto/ext/base/string_view.h" +#include "protos/third_party/simpleperf/record_file.pbzero.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/profiler_tables_py.h" +#include "src/trace_processor/types/trace_processor_context.h" + +namespace perfetto::trace_processor::perf_importer { +namespace { + +using third_party::simpleperf::proto::pbzero::FileFeature; +using DexFile = FileFeature::DexFile; +using ElfFile = FileFeature::ElfFile; +using KernelModule = FileFeature::KernelModule; +using DsoType = FileFeature::DsoType; +using Symbol = FileFeature::Symbol; + +void InsertSymbols(const FileFeature::Decoder& file, + AddressRangeMap<std::string>& out) { + for (auto raw_symbol = file.symbol(); raw_symbol; ++raw_symbol) { + Symbol::Decoder symbol(*raw_symbol); + out.DeleteOverlapsAndEmplace( + AddressRange::FromStartAndSize(symbol.vaddr(), symbol.len()), + symbol.name().ToStdString()); + } +} +} // namespace + +DsoTracker::DsoTracker(TraceProcessorContext* context) + : context_(context), + mapping_table_(context_->storage->stack_profile_mapping_table()) {} +DsoTracker::~DsoTracker() = default; + +void DsoTracker::AddSimpleperfFile2(const FileFeature::Decoder& file) { + Dso dso; + switch (file.type()) { + case DsoType::DSO_KERNEL: + InsertSymbols(file, kernel_symbols_); + return; + + case DsoType::DSO_ELF_FILE: { + ElfFile::Decoder elf(file.elf_file()); + dso.load_bias = file.min_vaddr() - elf.file_offset_of_min_vaddr(); + break; + } + + case DsoType::DSO_KERNEL_MODULE: { + KernelModule::Decoder module(file.kernel_module()); + dso.load_bias = file.min_vaddr() - module.memory_offset_of_min_vaddr(); + break; + } + + case DsoType::DSO_DEX_FILE: + case DsoType::DSO_SYMBOL_MAP_FILE: + case DsoType::DSO_UNKNOWN_FILE: + return; + } + + InsertSymbols(file, dso.symbols); + files_.Insert(context_->storage->InternString(file.path()), std::move(dso)); +} + +void DsoTracker::SymbolizeFrames() { + const StringId kEmptyString = context_->storage->InternString(""); + for (auto frame = context_->storage->mutable_stack_profile_frame_table() + ->IterateRows(); + frame; ++frame) { + if (frame.name() != kNullStringId && frame.name() != kEmptyString) { + continue; + } + + if (!TrySymbolizeFrame(frame.row_reference())) { + SymbolizeKernelFrame(frame.row_reference()); + } + } +} + +void DsoTracker::SymbolizeKernelFrame( + tables::StackProfileFrameTable::RowReference frame) { + const auto mapping = *mapping_table_.FindById(frame.mapping()); + uint64_t address = static_cast<uint64_t>(frame.rel_pc()) + + static_cast<uint64_t>(mapping.start()); + auto symbol = kernel_symbols_.Find(address); + if (symbol == kernel_symbols_.end()) { + return; + } + frame.set_name( + context_->storage->InternString(base::StringView(symbol->second))); +} + +bool DsoTracker::TrySymbolizeFrame( + tables::StackProfileFrameTable::RowReference frame) { + const auto mapping = *mapping_table_.FindById(frame.mapping()); + auto* file = files_.Find(mapping.name()); + if (!file) { + return false; + } + + // Load bias is something we can only determine by looking at the actual elf + // file. Thus PERF_RECORD_MMAP{2} events do not record it. So we need to + // potentially do an adjustment here if the load_bias tracked in the mapping + // table and the one reported by the file are mismatched. + uint64_t adj = file->load_bias - static_cast<uint64_t>(mapping.load_bias()); + + auto symbol = file->symbols.Find(static_cast<uint64_t>(frame.rel_pc()) + adj); + if (symbol == file->symbols.end()) { + return false; + } + frame.set_name( + context_->storage->InternString(base::StringView(symbol->second))); + return true; +} + +} // namespace perfetto::trace_processor::perf_importer diff --git a/src/trace_processor/importers/perf/dso_tracker.h b/src/trace_processor/importers/perf/dso_tracker.h new file mode 100644 index 000000000..314d54935 --- /dev/null +++ b/src/trace_processor/importers/perf/dso_tracker.h @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_DSO_TRACKER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_DSO_TRACKER_H_ + +#include <cstdint> +#include <string> + +#include "perfetto/base/status.h" +#include "perfetto/ext/base/flat_hash_map.h" +#include "protos/third_party/simpleperf/record_file.pbzero.h" +#include "src/trace_processor/importers/common/address_range.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/profiler_tables_py.h" +#include "src/trace_processor/types/destructible.h" +#include "src/trace_processor/types/trace_processor_context.h" + +namespace perfetto::trace_processor::perf_importer { + +// Keeps track of DSO symbols to symbolize frames at the end of the trace +// parsing. +// TODO(b/334978369): We could potentially use this class (or a similar one) to +// process the ModuleSymbols proto packets and consolidate all symbolization in +// one place. +class DsoTracker : public Destructible { + public: + static DsoTracker& GetOrCreate(TraceProcessorContext* context) { + if (!context->perf_dso_tracker) { + context->perf_dso_tracker.reset(new DsoTracker(context)); + } + return static_cast<DsoTracker&>(*context->perf_dso_tracker); + } + ~DsoTracker() override; + + // Add symbol data contained in a `FileFeature` proto. + void AddSimpleperfFile2( + const third_party::simpleperf::proto::pbzero::FileFeature::Decoder& file); + + // Tries to symbolize any `STACK_PROFILE_FRAME` frame missing the `name` + // attribute. This should be called at the end of parsing when all packets + // have been processed and all tables updated. + void SymbolizeFrames(); + + private: + struct Dso { + uint64_t load_bias; + AddressRangeMap<std::string> symbols; + }; + + explicit DsoTracker(TraceProcessorContext* context); + + void SymbolizeKernelFrame(tables::StackProfileFrameTable::RowReference frame); + // Returns true it the frame was symbolized. + bool TrySymbolizeFrame(tables::StackProfileFrameTable::RowReference frame); + + TraceProcessorContext* const context_; + const tables::StackProfileMappingTable& mapping_table_; + base::FlatHashMap<StringId, Dso> files_; + AddressRangeMap<std::string> kernel_symbols_; +}; + +} // namespace perfetto::trace_processor::perf_importer + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_DSO_TRACKER_H_ diff --git a/src/trace_processor/importers/perf/features.cc b/src/trace_processor/importers/perf/features.cc new file mode 100644 index 000000000..877f2fd09 --- /dev/null +++ b/src/trace_processor/importers/perf/features.cc @@ -0,0 +1,296 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/perf/features.h" + +#include <cstdint> +#include <utility> + +#include "perfetto/base/logging.h" +#include "perfetto/base/status.h" +#include "perfetto/ext/base/string_utils.h" +#include "perfetto/ext/base/string_view.h" +#include "perfetto/trace_processor/status.h" +#include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/importers/perf/perf_event.h" +#include "src/trace_processor/importers/perf/reader.h" +#include "src/trace_processor/util/status_macros.h" + +namespace perfetto::trace_processor::perf_importer::feature { +namespace { + +struct BuildIdRecord { + static constexpr uint8_t kMaxSize = 20; + char data[kMaxSize]; + uint8_t size; + uint8_t reserved[3]; +}; + +uint8_t CountTrailingZeros(const BuildIdRecord& build_id) { + for (uint8_t i = 0; i < BuildIdRecord::kMaxSize; ++i) { + if (build_id.data[BuildIdRecord::kMaxSize - i - 1] != 0) { + return i; + } + } + return sizeof(build_id.data); +} + +// BuildIds are usually SHA-1 hashes (20 bytes), sometimes MD5 (16 bytes), +// sometimes 8 bytes long. Simpleperf adds trailing zeros up to 20. Do a best +// guess based on the number of trailing zeros. +uint8_t GuessBuildIdSize(const BuildIdRecord& build_id) { + static_assert(BuildIdRecord::kMaxSize == 20); + uint8_t len = BuildIdRecord::kMaxSize - CountTrailingZeros(build_id); + if (len > 16) { + return BuildIdRecord::kMaxSize; + } + if (len > 8) { + return 16; + } + return 8; +} + +bool ParseString(Reader& reader, std::string& out) { + uint32_t len; + base::StringView str; + if (!reader.Read(len) || len == 0 || !reader.ReadStringView(str, len)) { + return false; + } + + if (str.at(len - 1) != '\0') { + return false; + } + + // Strings are padded with null values, stop at first null + out = std::string(str.data()); + return true; +} + +bool ParseBuildId(const perf_event_header& header, + TraceBlobView blob, + BuildId& out) { + Reader reader(std::move(blob)); + + BuildIdRecord build_id; + + if (!reader.Read(out.pid) || !reader.Read(build_id) || + !reader.ReadStringUntilEndOrNull(out.filename)) { + return false; + } + + if (header.misc & PERF_RECORD_MISC_EXT_RESERVED) { + if (build_id.size > BuildIdRecord::kMaxSize) { + return false; + } + } else { + // Probably a simpleperf trace. Simpleperf fills build_ids with zeros up + // to a length of 20 and leaves the rest uninitialized :( so we can not read + // build_id.size or build_id.reserved to do any checks. + // TODO(b/334978369): We should be able to tell for sure whether this is + // simpleperf or not by checking the existence of SimpleperfMetaInfo. + build_id.size = GuessBuildIdSize(build_id); + } + out.build_id = std::string(build_id.data, build_id.size); + return true; +} + +util::Status ParseEventTypeInfo(std::string value, SimpleperfMetaInfo& out) { + for (const auto& line : base::SplitString(value, "\n")) { + auto tokens = base::SplitString(line, ","); + if (tokens.size() != 3) { + return util::ErrStatus("Invalid event_type_info: '%s'", line.c_str()); + } + + auto type = base::StringToUInt32(tokens[1]); + if (!type) { + return util::ErrStatus("Could not parse type in event_type_info: '%s'", + tokens[1].c_str()); + } + auto config = base::StringToUInt64(tokens[2]); + if (!config) { + return util::ErrStatus("Could not parse config in event_type_info: '%s'", + tokens[2].c_str()); + } + + out.event_type_info.Insert({*type, *config}, std::move(tokens[0])); + } + + return util::OkStatus(); +} + +util::Status ParseSimpleperfMetaInfoEntry( + std::pair<std::string, std::string> entry, + SimpleperfMetaInfo& out) { + static constexpr char kEventTypeInfoKey[] = "event_type_info"; + if (entry.first == kEventTypeInfoKey) { + return ParseEventTypeInfo(std::move(entry.second), out); + } + + PERFETTO_CHECK( + out.entries.Insert(std::move(entry.first), std::move(entry.second)) + .second); + return util::OkStatus(); +} + +} // namespace + +// static +util::Status BuildId::Parse(TraceBlobView bytes, + std::function<util::Status(BuildId)> cb) { + Reader reader(std::move(bytes)); + while (reader.size_left() != 0) { + perf_event_header header; + TraceBlobView payload; + if (!reader.Read(header)) { + return base::ErrStatus( + "Failed to parse feature BuildId. Could not read header."); + } + if (header.size < sizeof(header)) { + return base::ErrStatus( + "Failed to parse feature BuildId. Invalid size in header."); + } + if (!reader.ReadBlob(payload, header.size - sizeof(header))) { + return base::ErrStatus( + "Failed to parse feature BuildId. Could not read payload."); + } + + BuildId build_id; + if (!ParseBuildId(header, std::move(payload), build_id)) { + return base::ErrStatus( + "Failed to parse feature BuildId. Could not read entry."); + } + + RETURN_IF_ERROR(cb(std::move(build_id))); + } + return util::OkStatus(); +} + +// static +util::Status SimpleperfMetaInfo::Parse(const TraceBlobView& bytes, + SimpleperfMetaInfo& out) { + auto* it_end = reinterpret_cast<const char*>(bytes.data() + bytes.size()); + for (auto* it = reinterpret_cast<const char*>(bytes.data()); it != it_end;) { + auto end = std::find(it, it_end, '\0'); + if (end == it_end) { + return util::ErrStatus("Failed to read key from Simpleperf MetaInfo"); + } + std::string key(it, end); + it = end; + ++it; + if (it == it_end) { + return util::ErrStatus("Missing value in Simpleperf MetaInfo"); + } + end = std::find(it, it_end, '\0'); + if (end == it_end) { + return util::ErrStatus("Failed to read value from Simpleperf MetaInfo"); + } + std::string value(it, end); + it = end; + ++it; + + RETURN_IF_ERROR(ParseSimpleperfMetaInfoEntry( + std::make_pair(std::move(key), std::move(value)), out)); + } + return util::OkStatus(); +} + +// static +util::Status EventDescription::Parse( + TraceBlobView bytes, + std::function<util::Status(EventDescription)> cb) { + Reader reader(std::move(bytes)); + uint32_t nr; + uint32_t attr_size; + if (!reader.Read(nr) || !reader.Read(attr_size)) { + return util::ErrStatus("Failed to parse header for PERF_EVENT_DESC"); + } + + for (; nr != 0; --nr) { + EventDescription desc; + uint32_t nr_ids; + if (!reader.ReadPerfEventAttr(desc.attr, attr_size) || + !reader.Read(nr_ids) || !ParseString(reader, desc.event_string)) { + return util::ErrStatus("Failed to parse record for PERF_EVENT_DESC"); + } + + desc.ids.resize(nr_ids); + for (uint64_t& id : desc.ids) { + if (!reader.Read(id)) { + return util::ErrStatus("Failed to parse ids for PERF_EVENT_DESC"); + } + } + RETURN_IF_ERROR(cb(std::move(desc))); + } + return util::OkStatus(); +} + +util::Status ParseSimpleperfFile2(TraceBlobView bytes, + std::function<void(TraceBlobView)> cb) { + Reader reader(std::move(bytes)); + while (reader.size_left() != 0) { + uint32_t len; + if (!reader.Read(len)) { + return base::ErrStatus("Failed to parse len in FEATURE_SIMPLEPERF_FILE2"); + } + TraceBlobView payload; + if (!reader.ReadBlob(payload, len)) { + return base::ErrStatus( + "Failed to parse payload in FEATURE_SIMPLEPERF_FILE2"); + } + cb(std::move(payload)); + } + return util::OkStatus(); +} + +// static +util::Status HeaderGroupDesc::Parse(TraceBlobView bytes, HeaderGroupDesc& out) { + Reader reader(std::move(bytes)); + uint32_t nr; + if (!reader.Read(nr)) { + return util::ErrStatus("Failed to parse header for HEADER_GROUP_DESC"); + } + + HeaderGroupDesc group_desc; + group_desc.entries.resize(nr); + for (auto& e : group_desc.entries) { + if (!ParseString(reader, e.string) || !reader.Read(e.leader_idx) || + !reader.Read(e.nr_members)) { + return util::ErrStatus("Failed to parse HEADER_GROUP_DESC entry"); + } + } + out = std::move(group_desc); + return base::OkStatus(); +} + +base::StatusOr<std::vector<std::string>> ParseCmdline(TraceBlobView bytes) { + Reader reader(std::move(bytes)); + uint32_t nr; + if (!reader.Read(nr)) { + return util::ErrStatus("Failed to parse nr for CMDLINE"); + } + + std::vector<std::string> args; + args.reserve(nr); + for (; nr != 0; --nr) { + args.emplace_back(); + if (!ParseString(reader, args.back())) { + return base::ErrStatus("Failed to parse string for CMDLINE"); + } + } + return std::move(args); +} + +} // namespace perfetto::trace_processor::perf_importer::feature diff --git a/src/trace_processor/importers/perf/features.h b/src/trace_processor/importers/perf/features.h new file mode 100644 index 000000000..e02895727 --- /dev/null +++ b/src/trace_processor/importers/perf/features.h @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_FEATURES_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_FEATURES_H_ + +#include <cstdint> +#include <functional> +#include <limits> +#include <string> +#include <vector> + +#include "perfetto/ext/base/flat_hash_map.h" +#include "perfetto/ext/base/hash.h" +#include "perfetto/ext/base/status_or.h" +#include "perfetto/trace_processor/status.h" +#include "src/trace_processor/importers/perf/perf_event.h" + +namespace perfetto ::trace_processor { +class TraceBlobView; + +namespace perf_importer::feature { + +enum Id : uint8_t { + ID_RESERVED = 0, + ID_TRACING_DATA = 1, + ID_BUILD_ID = 2, + ID_HOSTNAME = 3, + ID_OS_RELEASE = 4, + ID_VERSION = 5, + ID_ARCH = 6, + ID_NR_CPUS = 7, + ID_CPU_DESC = 8, + ID_CPU_ID = 9, + ID_TOTAL_MEM = 10, + ID_CMD_LINE = 11, + ID_EVENT_DESC = 12, + ID_CPU_TOPOLOGY = 13, + ID_NUMA_TOPOLOGY = 14, + ID_BRANCH_STACK = 15, + ID_PMU_MAPPINGS = 16, + ID_GROUP_DESC = 17, + ID_AUX_TRACE = 18, + ID_STAT = 19, + ID_CACHE = 20, + ID_SAMPLE_TIME = 21, + ID_SAMPLE_TOPOLOGY = 22, + ID_CLOCK_ID = 23, + ID_DIR_FORMAT = 24, + ID_BPF_PROG_INFO = 25, + ID_BPF_BTF = 26, + ID_COMPRESSED = 27, + ID_CPU_PUM_CAPS = 28, + ID_CLOCK_DATA = 29, + ID_HYBRID_TOPOLOGY = 30, + ID_PMU_CAPS = 31, + ID_SIMPLEPERF_FILE = 128, + ID_SIMPLEPERF_META_INFO = 129, + ID_SIMPLEPERF_FILE2 = 132, + ID_MAX = std::numeric_limits<uint8_t>::max(), +}; + +struct BuildId { + static util::Status Parse(TraceBlobView, + std::function<util::Status(BuildId)> cb); + int32_t pid; + std::string build_id; + std::string filename; +}; + +struct HeaderGroupDesc { + static util::Status Parse(TraceBlobView, HeaderGroupDesc& out); + struct Entry { + std::string string; + uint32_t leader_idx; + uint32_t nr_members; + }; + std::vector<Entry> entries; +}; + +struct EventDescription { + static util::Status Parse(TraceBlobView, + std::function<util::Status(EventDescription)> cb); + perf_event_attr attr; + std::string event_string; + std::vector<uint64_t> ids; +}; + +struct SimpleperfMetaInfo { + static util::Status Parse(const TraceBlobView&, SimpleperfMetaInfo& out); + base::FlatHashMap<std::string, std::string> entries; + struct EventTypeAndConfig { + uint32_t type; + uint64_t config; + bool operator==(const EventTypeAndConfig& other) { + return type == other.type && config == other.config; + } + bool operator!=(const EventTypeAndConfig& other) { + return !(*this == other); + } + struct Hasher { + size_t operator()(const EventTypeAndConfig& o) const { + return static_cast<size_t>(base::Hasher::Combine(o.config, o.type)); + } + }; + }; + using EventName = std::string; + base::FlatHashMap<EventTypeAndConfig, EventName, EventTypeAndConfig::Hasher> + event_type_info; +}; + +util::Status ParseSimpleperfFile2(TraceBlobView, + std::function<void(TraceBlobView)> cb); + +base::StatusOr<std::vector<std::string>> ParseCmdline(TraceBlobView blob); + +} // namespace perf_importer::feature + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_FEATURES_H_ diff --git a/src/trace_processor/importers/perf/mmap_record.cc b/src/trace_processor/importers/perf/mmap_record.cc new file mode 100644 index 000000000..d11fdd541 --- /dev/null +++ b/src/trace_processor/importers/perf/mmap_record.cc @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/perf/mmap_record.h" + +#include <optional> + +#include "perfetto/base/status.h" +#include "src/trace_processor/importers/perf/reader.h" +#include "src/trace_processor/importers/perf/record.h" + +namespace perfetto::trace_processor::perf_importer { + +base::Status MmapRecord::Parse(const Record& record) { + Reader reader(record.payload.copy()); + if (!reader.Read(*static_cast<CommonMmapRecordFields*>(this)) || + !reader.ReadCString(filename)) { + return base::ErrStatus("Failed to parse MMAP record"); + } + cpu_mode = record.GetCpuMode(); + return base::OkStatus(); +} + +base::Status Mmap2Record::Parse(const Record& record) { + Reader reader(record.payload.copy()); + if (!reader.Read(*static_cast<BaseMmap2Record*>(this)) || + !reader.ReadCString(filename)) { + return base::ErrStatus("Failed to parse MMAP record"); + } + + has_build_id = record.mmap_has_build_id(); + + if (has_build_id && build_id.build_id_size > + BaseMmap2Record::BuildIdFields::kMaxBuildIdSize) { + return base::ErrStatus( + "Invalid build_id_size in MMAP2 record. Expected <= %zu but found " + "%" PRIu8, + BaseMmap2Record::BuildIdFields::kMaxBuildIdSize, + build_id.build_id_size); + } + + cpu_mode = record.GetCpuMode(); + + return base::OkStatus(); +} + +std::optional<BuildId> Mmap2Record::GetBuildId() const { + return has_build_id ? std::make_optional(BuildId::FromRaw(std::string( + build_id.build_id_buf, build_id.build_id_size))) + : std::nullopt; +} + +} // namespace perfetto::trace_processor::perf_importer diff --git a/src/trace_processor/importers/perf/mmap_record.h b/src/trace_processor/importers/perf/mmap_record.h new file mode 100644 index 000000000..37b393942 --- /dev/null +++ b/src/trace_processor/importers/perf/mmap_record.h @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_MMAP_RECORD_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_MMAP_RECORD_H_ + +#include <cstdint> +#include <optional> +#include <string> +#include "perfetto/base/status.h" +#include "protos/perfetto/trace/profiling/profile_packet.pbzero.h" +#include "src/trace_processor/util/build_id.h" + +namespace perfetto::trace_processor::perf_importer { + +struct Record; + +struct CommonMmapRecordFields { + uint32_t pid; + uint32_t tid; + uint64_t addr; + uint64_t len; + uint64_t pgoff; +}; + +struct MmapRecord : public CommonMmapRecordFields { + std::string filename; + protos::pbzero::Profiling::CpuMode cpu_mode; + + base::Status Parse(const Record& record); +}; + +struct BaseMmap2Record : public CommonMmapRecordFields { + struct BuildIdFields { + static constexpr size_t kMaxBuildIdSize = 20; + uint8_t build_id_size; + uint8_t reserved_1; + uint16_t reserved_2; + char build_id_buf[kMaxBuildIdSize]; + }; + struct InodeFields { + uint32_t maj; + uint32_t min; + int64_t ino; + uint64_t ino_generation; + }; + static_assert(sizeof(BuildIdFields) == sizeof(InodeFields)); + + union { + BuildIdFields build_id; + InodeFields inode; + }; + uint32_t prot; + uint32_t flags; +}; + +struct Mmap2Record : public BaseMmap2Record { + std::string filename; + protos::pbzero::Profiling::CpuMode cpu_mode; + bool has_build_id; + + base::Status Parse(const Record& record); + std::optional<BuildId> GetBuildId() const; +}; + +} // namespace perfetto::trace_processor::perf_importer + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_MMAP_RECORD_H_ diff --git a/src/trace_processor/importers/perf/perf_counter.cc b/src/trace_processor/importers/perf/perf_counter.cc new file mode 100644 index 000000000..685e94040 --- /dev/null +++ b/src/trace_processor/importers/perf/perf_counter.cc @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/perf/perf_counter.h" + +#include <cstdint> + +#include "perfetto/base/logging.h" +#include "src/trace_processor/tables/counter_tables_py.h" + +namespace perfetto::trace_processor::perf_importer { + +void PerfCounter::AddDelta(int64_t ts, double delta) { + last_count_ += delta; + counter_table_.Insert({ts, track_id_, last_count_}); +} + +void PerfCounter::AddCount(int64_t ts, double count) { + PERFETTO_CHECK(count >= last_count_); + last_count_ = count; + counter_table_.Insert({ts, track_id_, last_count_}); +} + +} // namespace perfetto::trace_processor::perf_importer diff --git a/src/trace_processor/importers/perf/perf_counter.h b/src/trace_processor/importers/perf/perf_counter.h new file mode 100644 index 000000000..173620ef8 --- /dev/null +++ b/src/trace_processor/importers/perf/perf_counter.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_COUNTER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_COUNTER_H_ + +#include <cstdint> + +#include "src/trace_processor/tables/counter_tables_py.h" +#include "src/trace_processor/tables/profiler_tables_py.h" + +namespace perfetto::trace_processor::perf_importer { + +// Helper class to keep track of perf counters and convert delta values found in +// perf files to absolute values needed for the perfetto counter table. +class PerfCounter { + public: + PerfCounter(tables::CounterTable* counter_table, + const tables::PerfCounterTrackTable::ConstRowReference& track) + : counter_table_(*counter_table), + track_id_(track.id()), + is_timebase_(track.is_timebase()) {} + + bool is_timebase() const { return is_timebase_; } + + void AddDelta(int64_t ts, double delta); + void AddCount(int64_t ts, double count); + + private: + tables::CounterTable& counter_table_; + tables::PerfCounterTrackTable::Id track_id_; + const bool is_timebase_; + double last_count_{0}; +}; + +} // namespace perfetto::trace_processor::perf_importer + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_COUNTER_H_ diff --git a/src/trace_processor/importers/perf/perf_data_parser.cc b/src/trace_processor/importers/perf/perf_data_parser.cc deleted file mode 100644 index e3dfef308..000000000 --- a/src/trace_processor/importers/perf/perf_data_parser.cc +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/trace_processor/importers/perf/perf_data_parser.h" - -#include <optional> -#include <string> -#include <vector> -#include "perfetto/base/logging.h" -#include "perfetto/ext/base/string_utils.h" -#include "perfetto/trace_processor/trace_blob_view.h" -#include "src/trace_processor/importers/common/mapping_tracker.h" -#include "src/trace_processor/importers/common/process_tracker.h" -#include "src/trace_processor/importers/perf/perf_data_reader.h" -#include "src/trace_processor/importers/perf/perf_data_tracker.h" -#include "src/trace_processor/storage/trace_storage.h" -#include "src/trace_processor/tables/profiler_tables_py.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { - -using FramesTable = tables::StackProfileFrameTable; -using CallsitesTable = tables::StackProfileCallsiteTable; - -PerfDataParser::PerfDataParser(TraceProcessorContext* context) - : context_(context), tracker_(PerfDataTracker::GetOrCreate(context_)) {} - -PerfDataParser::~PerfDataParser() = default; - -base::StatusOr<PerfDataTracker::PerfSample> PerfDataParser::ParseSample( - TraceBlobView tbv) { - perf_importer::PerfDataReader reader(std::move(tbv)); - return tracker_->ParseSample(reader); -} - -void PerfDataParser::ParsePerfRecord(int64_t ts, TraceBlobView tbv) { - auto sample_status = ParseSample(std::move(tbv)); - if (!sample_status.ok()) { - return; - } - PerfDataTracker::PerfSample sample = *sample_status; - - // The sample has been validated in tokenizer so callchain shouldn't be empty. - PERFETTO_CHECK(!sample.callchain.empty()); - - // First instruction pointer in the callchain should be from kernel space, so - // it shouldn't be available in mappings. - UniquePid upid = context_->process_tracker->GetOrCreateProcess(*sample.pid); - if (context_->mapping_tracker->FindUserMappingForAddress( - upid, sample.callchain.front())) { - context_->storage->IncrementStats(stats::perf_samples_skipped); - return; - } - - if (sample.callchain.size() == 1) { - context_->storage->IncrementStats(stats::perf_samples_skipped); - return; - } - - std::vector<FramesTable::Row> frame_rows; - for (uint32_t i = 1; i < sample.callchain.size(); i++) { - UserMemoryMapping* mapping = - context_->mapping_tracker->FindUserMappingForAddress( - upid, sample.callchain[i]); - if (!mapping) { - context_->storage->IncrementStats(stats::perf_samples_skipped); - return; - } - FramesTable::Row new_row; - std::string mock_name = - base::StackString<1024>( - "%" PRIu64, sample.callchain[i] - mapping->memory_range().start()) - .ToStdString(); - new_row.name = context_->storage->InternString(mock_name.c_str()); - new_row.mapping = mapping->mapping_id(); - new_row.rel_pc = - static_cast<int64_t>(mapping->ToRelativePc(sample.callchain[i])); - frame_rows.push_back(new_row); - } - - // Insert frames. We couldn't do it before as no frames should be added if the - // mapping couldn't be found for any of them. - const auto& frames = context_->storage->mutable_stack_profile_frame_table(); - std::vector<FramesTable::Id> frame_ids; - for (const auto& row : frame_rows) { - frame_ids.push_back(frames->Insert(row).id); - } - - // Insert callsites. - const auto& callsites = - context_->storage->mutable_stack_profile_callsite_table(); - - std::optional<CallsitesTable::Id> parent_callsite_id; - for (uint32_t i = 0; i < frame_ids.size(); i++) { - CallsitesTable::Row callsite_row; - callsite_row.frame_id = frame_ids[i]; - callsite_row.depth = i; - callsite_row.parent_id = parent_callsite_id; - parent_callsite_id = callsites->Insert(callsite_row).id; - } - - // Insert stack sample. - tables::PerfSampleTable::Row perf_sample_row; - perf_sample_row.callsite_id = parent_callsite_id; - perf_sample_row.ts = ts; - if (sample.cpu) { - perf_sample_row.cpu = *sample.cpu; - } - if (sample.tid) { - auto utid = context_->process_tracker->GetOrCreateThread(*sample.tid); - perf_sample_row.utid = utid; - } - context_->storage->mutable_perf_sample_table()->Insert(perf_sample_row); -} - -} // namespace perf_importer -} // namespace trace_processor -} // namespace perfetto diff --git a/src/trace_processor/importers/perf/perf_data_parser.h b/src/trace_processor/importers/perf/perf_data_parser.h deleted file mode 100644 index f2ab0a321..000000000 --- a/src/trace_processor/importers/perf/perf_data_parser.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_PARSER_H_ -#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_PARSER_H_ - -#include <stdint.h> - -#include "perfetto/base/compiler.h" -#include "perfetto/trace_processor/trace_blob_view.h" -#include "src/trace_processor/importers/common/trace_parser.h" -#include "src/trace_processor/importers/perf/perf_data_tracker.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { - -// Parses samples from perf.data files. -class PerfDataParser : public PerfRecordParser { - public: - explicit PerfDataParser(TraceProcessorContext*); - ~PerfDataParser() override; - - // The data in TraceBlobView has to be a perf.data sample. - void ParsePerfRecord(int64_t timestamp, TraceBlobView) override; - - private: - base::StatusOr<PerfDataTracker::PerfSample> ParseSample(TraceBlobView); - - TraceProcessorContext* context_ = nullptr; - PerfDataTracker* tracker_ = nullptr; -}; - -} // namespace perf_importer -} // namespace trace_processor -} // namespace perfetto - -#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_PARSER_H_ diff --git a/src/trace_processor/importers/perf/perf_data_reader.cc b/src/trace_processor/importers/perf/perf_data_reader.cc deleted file mode 100644 index e0618266a..000000000 --- a/src/trace_processor/importers/perf/perf_data_reader.cc +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/trace_processor/importers/perf/perf_data_reader.h" - -#include <cstddef> -#include <optional> -#include <vector> -#include "perfetto/base/logging.h" -#include "perfetto/trace_processor/trace_blob_view.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { -void PerfDataReader::SkipSlow(size_t bytes_to_skip) { - size_t bytes_in_buffer = BytesInBuffer(); - - // Size fits in buffer. - if (bytes_in_buffer >= bytes_to_skip) { - buffer_offset_ += bytes_to_skip; - return; - } - - // Empty the buffer and increase the |blob_offset_|. - buffer_offset_ = 0; - buffer_.clear(); - blob_offset_ += bytes_to_skip - bytes_in_buffer; -} - -void PerfDataReader::PeekSlow(uint8_t* obj_data, size_t size) const { - size_t bytes_in_buffer = BytesInBuffer(); - - // Read from buffer. - if (bytes_in_buffer >= size) { - memcpy(obj_data, buffer_.data() + buffer_offset_, size); - return; - } - - // Read from blob and buffer. - memcpy(obj_data, buffer_.data() + buffer_offset_, bytes_in_buffer); - memcpy(obj_data + bytes_in_buffer, tbv_.data() + blob_offset_, - size - bytes_in_buffer); -} - -TraceBlobView PerfDataReader::PeekTraceBlobViewSlow(size_t size) const { - auto blob = TraceBlob::Allocate(size); - size_t bytes_in_buffer = BytesInBuffer(); - - // Data is in buffer, so we need to create a new TraceBlob from it. - if (bytes_in_buffer >= size) { - memcpy(blob.data(), buffer_.data() + buffer_offset_, size); - return TraceBlobView(std::move(blob)); - } - - // Data is in between blob and buffer and we need to dump data from buffer - // and blob to a new TraceBlob. - size_t bytes_from_blob = size - bytes_in_buffer; - memcpy(blob.data(), buffer_.data() + buffer_offset_, bytes_in_buffer); - memcpy(blob.data() + bytes_in_buffer, tbv_.data() + blob_offset_, - bytes_from_blob); - return TraceBlobView(std::move(blob)); -} - -} // namespace perf_importer -} // namespace trace_processor -} // namespace perfetto diff --git a/src/trace_processor/importers/perf/perf_data_reader.h b/src/trace_processor/importers/perf/perf_data_reader.h deleted file mode 100644 index acf07d1e6..000000000 --- a/src/trace_processor/importers/perf/perf_data_reader.h +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_READER_H_ -#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_READER_H_ - -#include <stdint.h> -#include <cstddef> -#include <cstring> -#include <optional> -#include <vector> - -#include "perfetto/base/logging.h" -#include "perfetto/trace_processor/trace_blob.h" -#include "perfetto/trace_processor/trace_blob_view.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { - -// Reader class for tokenizing and parsing. Currently used by perf importer, but -// it's design is not related to perf. Responsible for hiding away the -// complexity of reading values from TraceBlobView and glueing the tbvs together -// in case there is data between many of them. -class PerfDataReader { - public: - PerfDataReader() = default; - explicit PerfDataReader(TraceBlobView tbv) : tbv_(std::move(tbv)) {} - - // Updates old TraceBlobView with new one. If there is data left in the old - // one, it will be saved in the buffer. - void Append(TraceBlobView tbv) { - uint64_t size_before = BytesAvailable(); - buffer_.insert(buffer_.end(), tbv_.data() + blob_offset_, - tbv_.data() + tbv_.size()); - tbv_ = std::move(tbv); - blob_offset_ = 0; - - // Post condition. Checks whether no data has been lost in the Append. - PERFETTO_DCHECK(BytesAvailable() == size_before + tbv.size()); - } - - // Reads the |obj| and updates |file_offset_| of the reader. - // NOTE: Assumes count of bytes available is higher than sizeof(T). - template <typename T> - void Read(T& obj) { - Peek(obj); - Skip<T>(); - } - - // Reads the T value for std::optional<T>. - // NOTE: Assumes count of bytes available is higher than sizeof(T). - template <typename T> - void ReadOptional(std::optional<T>& obj) { - T val; - Read(val); - obj = val; - } - - // Reads all of the data in the |vec| and updates |file_offset_| of the - // reader. - // NOTE: Assumes count of bytes available is higher than sizeof(T). - template <typename T> - void ReadVector(std::vector<T>& vec) { - PERFETTO_DCHECK(CanReadSize(sizeof(T) * vec.size())); - for (T& val : vec) { - Read(val); - } - } - - // Updates the |file_offset_| by the sizeof(T). - // NOTE: Assumes count of bytes available is higher than sizeof(T). - template <typename T> - void Skip() { - Skip(sizeof(T)); - } - - // Updates the |file_offset_| by the |bytes_to_skip|. - // NOTE: Assumes count of bytes available is higher than sizeof(T). - void Skip(uint64_t bytes_to_skip) { - uint64_t bytes_available_before = BytesAvailable(); - PERFETTO_DCHECK(CanReadSize(bytes_to_skip)); - size_t skip = static_cast<size_t>(bytes_to_skip); - - // Incrementing file offset is not related to the way data is split. - file_offset_ += skip; - size_t bytes_in_buffer = BytesInBuffer(); - - // Empty buffer. Increment |blob_offset_|. - if (PERFETTO_LIKELY(bytes_in_buffer == 0)) { - buffer_offset_ = 0; - buffer_.clear(); - blob_offset_ += skip; - } else { - SkipSlow(skip); - } - PERFETTO_DCHECK(BytesAvailable() == bytes_available_before - skip); - } - - // Peeks the |obj| without updating the |file_offset_| of the reader. - // NOTE: Assumes count of bytes available is higher than sizeof(T). - template <typename T> - void Peek(T& obj) const { - PERFETTO_DCHECK(CanReadSize(sizeof(T))); - size_t bytes_available_before = BytesAvailable(); - - // Read from blob. - if (PERFETTO_LIKELY(BytesInBuffer() == 0)) { - memcpy(&obj, tbv_.data() + blob_offset_, sizeof(T)); - } else { - PeekSlow(reinterpret_cast<uint8_t*>(&obj), sizeof(T)); - } - - PERFETTO_DCHECK(BytesAvailable() == bytes_available_before); - } - - // Creates TraceBlobView with data of |data_size| bytes from current offset. - // NOTE: Assumes count of bytes available is higher than sizeof(T). - TraceBlobView PeekTraceBlobView(uint64_t data_size) const { - PERFETTO_DCHECK(CanReadSize(data_size)); - size_t size = static_cast<size_t>(data_size); - size_t bytes_in_buffer = BytesInBuffer(); - - // Data is in blob, so it's enough to slice the existing |tbv_|. - if (PERFETTO_LIKELY(bytes_in_buffer == 0)) { - return tbv_.slice(tbv_.data() + blob_offset_, size); - } - return PeekTraceBlobViewSlow(size); - } - - // Returns if there is enough data to read offsets between |start| and |end|. - bool CanAccessFileRange(uint64_t start, uint64_t end) const { - return CanAccessFileOffset(static_cast<size_t>(start)) && - CanAccessFileOffset(static_cast<size_t>(end)); - } - - // Returns if there is enough data to read |size| bytes. - bool CanReadSize(uint64_t size) const { return size <= BytesAvailable(); } - - uint64_t current_file_offset() const { return file_offset_; } - - private: - void SkipSlow(size_t bytes_to_skip); - - void PeekSlow(uint8_t* obj_data, size_t) const; - - TraceBlobView PeekTraceBlobViewSlow(size_t) const; - - size_t BytesInBuffer() const { - PERFETTO_DCHECK(buffer_.size() >= buffer_offset_); - return buffer_.size() - buffer_offset_; - } - size_t BytesInBlob() const { return tbv_.size() - blob_offset_; } - size_t BytesAvailable() const { return BytesInBuffer() + BytesInBlob(); } - - bool CanAccessFileOffset(size_t off) const { - return off >= file_offset_ && off <= file_offset_ + BytesAvailable(); - } - - TraceBlobView tbv_; - std::vector<uint8_t> buffer_; - - // Where we are in relation to the current blob. - size_t blob_offset_ = 0; - // Where we are in relation to the file. - size_t file_offset_ = 0; - // Where we are in relation to the buffer. - size_t buffer_offset_ = 0; -}; -} // namespace perf_importer -} // namespace trace_processor -} // namespace perfetto - -#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_READER_H_ diff --git a/src/trace_processor/importers/perf/perf_data_reader_unittest.cc b/src/trace_processor/importers/perf/perf_data_reader_unittest.cc deleted file mode 100644 index 5bf30810d..000000000 --- a/src/trace_processor/importers/perf/perf_data_reader_unittest.cc +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/trace_processor/importers/perf/perf_data_reader.h" - -#include <stddef.h> - -#include "perfetto/base/build_config.h" -#include "test/gtest_and_gmock.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { - -namespace { -template <typename T> -TraceBlobView TraceBlobViewFromVector(std::vector<T> nums) { - size_t data_size = sizeof(T) * nums.size(); - auto blob = TraceBlob::Allocate(data_size); - memcpy(blob.data(), nums.data(), data_size); - return TraceBlobView(std::move(blob)); -} -} // namespace - -TEST(PerfDataReaderUnittest, AppendToEmpty) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{1, 2, 3}); - PerfDataReader reader; - EXPECT_FALSE(reader.CanReadSize(1)); - reader.Append(std::move(tbv)); - EXPECT_TRUE(reader.CanReadSize(sizeof(uint64_t) * 2)); -} - -TEST(PerfDataReaderUnittest, Append) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{1, 2, 3}); - PerfDataReader reader(std::move(tbv)); - - EXPECT_TRUE(reader.CanReadSize(sizeof(uint64_t) * 3)); - EXPECT_FALSE(reader.CanReadSize(sizeof(uint64_t) * 3 + 1)); - - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 2})); - EXPECT_TRUE(reader.CanReadSize(sizeof(uint64_t) * 5)); -} - -TEST(PerfDataReaderUnittest, Read) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 8}); - PerfDataReader reader(std::move(tbv)); - uint64_t val; - reader.Read(val); - EXPECT_EQ(val, 2u); -} - -TEST(PerfDataReaderUnittest, ReadFromBuffer) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 6}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3})); - - // Now the first vector should be in the buffer. - uint64_t val; - reader.Read(val); - EXPECT_EQ(val, 2u); -} - -TEST(PerfDataReaderUnittest, ReadBetweenBufferAndBlob) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3, 5})); - - struct Nums { - uint64_t x; - uint64_t y; - uint64_t z; - }; - - Nums nums; - reader.Read(nums); - - EXPECT_EQ(nums.x, 2u); - EXPECT_EQ(nums.y, 4u); - EXPECT_EQ(nums.z, 1u); -} - -TEST(PerfDataReaderUnittest, ReadOptional) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 8}); - PerfDataReader reader(std::move(tbv)); - std::optional<uint64_t> val; - reader.ReadOptional(val); - EXPECT_EQ(val, 2u); -} - -TEST(PerfDataReaderUnittest, ReadVector) { - TraceBlobView tbv = - TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 8, 16, 32}); - PerfDataReader reader(std::move(tbv)); - - std::vector<uint64_t> res(3); - reader.ReadVector(res); - - std::vector<uint64_t> valid{2, 4, 8}; - EXPECT_EQ(res, valid); -} - -TEST(PerfDataReaderUnittest, Skip) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 8}); - PerfDataReader reader(std::move(tbv)); - - reader.Skip<uint64_t>(); - - uint64_t val; - reader.Read(val); - EXPECT_EQ(val, 4u); -} - -TEST(PerfDataReaderUnittest, SkipInBuffer) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3, 5})); - - reader.Skip<uint64_t>(); - EXPECT_EQ(reader.current_file_offset(), sizeof(uint64_t)); -} - -TEST(PerfDataReaderUnittest, SkipBetweenBufferAndBlob) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3, 5})); - - struct Nums { - uint64_t x; - uint64_t y; - uint64_t z; - }; - - reader.Skip<Nums>(); - EXPECT_EQ(reader.current_file_offset(), sizeof(Nums)); -} - -TEST(PerfDataReaderUnittest, Peek) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 8}); - PerfDataReader reader(std::move(tbv)); - - uint64_t peek_val; - reader.Peek(peek_val); - - uint64_t val; - reader.Read(val); - EXPECT_EQ(val, 2u); -} - -TEST(PerfDataReaderUnittest, PeekFromBuffer) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 6}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3})); - - uint64_t val; - reader.Peek(val); - EXPECT_EQ(val, 2u); -} - -TEST(PerfDataReaderUnittest, PeekBetweenBufferAndBlob) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3, 5})); - - struct Nums { - uint64_t x; - uint64_t y; - uint64_t z; - }; - - Nums nums; - reader.Peek(nums); - - EXPECT_EQ(nums.x, 2u); - EXPECT_EQ(nums.y, 4u); - EXPECT_EQ(nums.z, 1u); -} - -TEST(PerfDataReaderUnittest, GetTraceBlobView) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 8}); - PerfDataReader reader(std::move(tbv)); - EXPECT_TRUE(reader.CanReadSize(sizeof(uint64_t) * 3)); - - TraceBlobView new_tbv = reader.PeekTraceBlobView(sizeof(uint64_t) * 2); - PerfDataReader new_reader(std::move(new_tbv)); - EXPECT_TRUE(new_reader.CanReadSize(sizeof(uint64_t) * 2)); - EXPECT_FALSE(new_reader.CanReadSize(sizeof(uint64_t) * 3)); -} - -TEST(PerfDataReaderUnittest, GetTraceBlobViewFromBuffer) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3, 5})); - - TraceBlobView new_tbv = reader.PeekTraceBlobView(sizeof(uint64_t) * 2); - PerfDataReader new_reader(std::move(new_tbv)); - EXPECT_TRUE(new_reader.CanReadSize(sizeof(uint64_t) * 2)); - EXPECT_FALSE(new_reader.CanReadSize(sizeof(uint64_t) * 3)); -} - -TEST(PerfDataReaderUnittest, GetTraceBlobViewFromBetweenBufferAndBlob) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4}); - PerfDataReader reader(std::move(tbv)); - reader.Append(TraceBlobViewFromVector(std::vector<uint64_t>{1, 3, 5})); - - TraceBlobView new_tbv = reader.PeekTraceBlobView(sizeof(uint64_t) * 3); - PerfDataReader new_reader(std::move(new_tbv)); - EXPECT_TRUE(new_reader.CanReadSize(sizeof(uint64_t) * 3)); - EXPECT_FALSE(new_reader.CanReadSize(sizeof(uint64_t) * 4)); -} - -TEST(PerfDataReaderUnittest, CanAccessFileRange) { - TraceBlobView tbv = TraceBlobViewFromVector(std::vector<uint64_t>{2, 4, 8}); - PerfDataReader reader(std::move(tbv)); - EXPECT_TRUE(reader.CanAccessFileRange(2, sizeof(uint64_t) * 3)); - EXPECT_FALSE(reader.CanAccessFileRange(2, sizeof(uint64_t) * 3 + 10)); -} - -} // namespace perf_importer - -} // namespace trace_processor -} // namespace perfetto diff --git a/src/trace_processor/importers/perf/perf_data_tokenizer.cc b/src/trace_processor/importers/perf/perf_data_tokenizer.cc index 25b54b983..4fd64b5f2 100644 --- a/src/trace_processor/importers/perf/perf_data_tokenizer.cc +++ b/src/trace_processor/importers/perf/perf_data_tokenizer.cc @@ -16,56 +16,101 @@ #include "src/trace_processor/importers/perf/perf_data_tokenizer.h" +#include <cstddef> #include <cstdint> #include <cstring> +#include <optional> +#include <string> +#include <utility> #include <vector> +#include "perfetto/base/flat_set.h" #include "perfetto/base/logging.h" #include "perfetto/base/status.h" #include "perfetto/ext/base/status_or.h" +#include "perfetto/public/compiler.h" #include "perfetto/trace_processor/trace_blob_view.h" +#include "protos/perfetto/trace/clock_snapshot.pbzero.h" +#include "protos/third_party/simpleperf/record_file.pbzero.h" +#include "src/trace_processor/importers/common/clock_tracker.h" #include "src/trace_processor/importers/common/slice_tracker.h" -#include "src/trace_processor/importers/perf/perf_data_reader.h" -#include "src/trace_processor/importers/perf/perf_data_tracker.h" +#include "src/trace_processor/importers/perf/attrs_section_reader.h" +#include "src/trace_processor/importers/perf/dso_tracker.h" +#include "src/trace_processor/importers/perf/features.h" #include "src/trace_processor/importers/perf/perf_event.h" +#include "src/trace_processor/importers/perf/perf_file.h" +#include "src/trace_processor/importers/perf/perf_session.h" +#include "src/trace_processor/importers/perf/reader.h" +#include "src/trace_processor/importers/perf/record.h" +#include "src/trace_processor/importers/proto/perf_sample_tracker.h" #include "src/trace_processor/sorter/trace_sorter.h" #include "src/trace_processor/storage/stats.h" +#include "src/trace_processor/util/build_id.h" #include "src/trace_processor/util/status_macros.h" -#include "protos/perfetto/trace/profiling/profile_packet.pbzero.h" - namespace perfetto { namespace trace_processor { namespace perf_importer { namespace { -protos::pbzero::Profiling::CpuMode GetCpuMode(const perf_event_header& header) { - switch (header.misc & kPerfRecordMiscCpumodeMask) { - case PERF_RECORD_MISC_KERNEL: - return protos::pbzero::Profiling::MODE_KERNEL; - case PERF_RECORD_MISC_USER: - return protos::pbzero::Profiling::MODE_USER; - case PERF_RECORD_MISC_HYPERVISOR: - return protos::pbzero::Profiling::MODE_HYPERVISOR; - case PERF_RECORD_MISC_GUEST_KERNEL: - return protos::pbzero::Profiling::MODE_GUEST_KERNEL; - case PERF_RECORD_MISC_GUEST_USER: - return protos::pbzero::Profiling::MODE_GUEST_USER; - default: - return protos::pbzero::Profiling::MODE_UNKNOWN; + +void AddIds(uint8_t id_offset, + uint64_t flags, + base::FlatSet<uint8_t>& feature_ids) { + for (size_t i = 0; i < sizeof(flags) * 8; ++i) { + if (flags & 1) { + feature_ids.insert(id_offset); + } + flags >>= 1; + ++id_offset; } } + +base::FlatSet<uint8_t> ExtractFeatureIds(const uint64_t& flags, + const uint64_t (&flags1)[3]) { + base::FlatSet<uint8_t> feature_ids; + AddIds(0, flags, feature_ids); + AddIds(64, flags1[0], feature_ids); + AddIds(128, flags1[1], feature_ids); + AddIds(192, flags1[2], feature_ids); + return feature_ids; +} + +bool ReadTime(const Record& record, std::optional<uint64_t>& time) { + if (!record.attr) { + time = std::nullopt; + return true; + } + Reader reader(record.payload.copy()); + if (record.header.type != PERF_RECORD_SAMPLE) { + std::optional<size_t> offset = record.attr->time_offset_from_end(); + if (!offset.has_value()) { + time = std::nullopt; + return true; + } + if (*offset > reader.size_left()) { + return false; + } + return reader.Skip(reader.size_left() - *offset) && + reader.ReadOptional(time); + } + + std::optional<size_t> offset = record.attr->time_offset_from_start(); + if (!offset.has_value()) { + time = std::nullopt; + return true; + } + return reader.Skip(*offset) && reader.ReadOptional(time); +} + } // namespace PerfDataTokenizer::PerfDataTokenizer(TraceProcessorContext* ctx) - : context_(ctx), - tracker_(PerfDataTracker::GetOrCreate(context_)), - reader_() {} + : context_(ctx) {} PerfDataTokenizer::~PerfDataTokenizer() = default; // A normal perf.data consts of: // [ header ] -// [ event ids (one array per attr) ] // [ attr section ] // [ data section ] // [ optional feature sections ] @@ -75,221 +120,323 @@ PerfDataTokenizer::~PerfDataTokenizer() = default; // Most file format documentation is outdated or misleading, instead see // perf_session__do_write_header() in linux/tools/perf/util/header.c. base::Status PerfDataTokenizer::Parse(TraceBlobView blob) { - reader_.Append(std::move(blob)); + buffer_.PushBack(std::move(blob)); - while (parsing_state_ != ParsingState::Records) { - base::StatusOr<ParsingResult> parsed = ParsingResult::Success; + base::StatusOr<ParsingResult> result = ParsingResult::kSuccess; + while (result.ok() && result.value() == ParsingResult::kSuccess && + !buffer_.empty()) { switch (parsing_state_) { - case ParsingState::Records: + case ParsingState::kParseHeader: + result = ParseHeader(); break; - case ParsingState::Header: - parsed = ParseHeader(); + + case ParsingState::kParseAttrs: + result = ParseAttrs(); break; - case ParsingState::AfterHeaderBuffer: - parsed = ParseAfterHeaderBuffer(); + + case ParsingState::kSeekRecords: + result = SeekRecords(); break; - case ParsingState::Attrs: - parsed = ParseAttrs(); + + case ParsingState::kParseRecords: + result = ParseRecords(); break; - case ParsingState::AttrIdsFromBuffer: - parsed = ParseAttrIdsFromBuffer(); + + case ParsingState::kParseFeatures: + result = ParseFeatures(); break; - case ParsingState::AttrIds: - parsed = ParseAttrIds(); + + case ParsingState::kParseFeatureSections: + result = ParseFeatureSections(); break; + + case ParsingState::kDone: + result = base::ErrStatus("Unexpected data"); } + } + return result.status(); +} - // There has been an error while parsing. - RETURN_IF_ERROR(parsed.status()); +base::StatusOr<PerfDataTokenizer::ParsingResult> +PerfDataTokenizer::ParseHeader() { + auto tbv = buffer_.SliceOff(0, sizeof(header_)); + if (!tbv) { + return ParsingResult::kMoreDataNeeded; + } + PERFETTO_CHECK(Reader(std::move(*tbv)).Read(header_)); - // There is not enough data to parse so we need to load another blob. - if (*parsed == ParsingResult::NoSpace) - return base::OkStatus(); + // TODO: Check for endianess (big endian will have letters reversed); + if (memcmp(header_.magic, PerfFile::kPerfMagic, + sizeof(PerfFile::kPerfMagic)) != 0) { + return util::ErrStatus("Invalid magic string"); } - while (reader_.current_file_offset() < header_.data.end()) { - // Make sure |perf_event_header| of the sample is available. - if (!reader_.CanReadSize(sizeof(perf_event_header))) { - return base::OkStatus(); - } + if (header_.size != sizeof(PerfFile::Header)) { + return util::ErrStatus("Failed to perf file header size. Expected %" PRIu64 + ", found %zu", + sizeof(PerfFile::Header)); + } - perf_event_header ev_header; - reader_.Peek(ev_header); - PERFETTO_CHECK(ev_header.size >= sizeof(perf_event_header)); + feature_ids_ = ExtractFeatureIds(header_.flags, header_.flags1); + feature_headers_section_ = {header_.data.end(), + feature_ids_.size() * sizeof(PerfFile::Section)}; + context_->clock_tracker->SetTraceTimeClock( + protos::pbzero::ClockSnapshot::Clock::MONOTONIC); - if (!reader_.CanReadSize(ev_header.size)) { - return base::OkStatus(); + PERFETTO_CHECK(buffer_.PopFrontUntil(sizeof(PerfFile::Header))); + parsing_state_ = ParsingState::kParseAttrs; + return ParsingResult::kSuccess; +} + +base::StatusOr<PerfDataTokenizer::ParsingResult> +PerfDataTokenizer::ParseAttrs() { + std::optional<TraceBlobView> tbv = + buffer_.SliceOff(header_.attrs.offset, header_.attrs.size); + if (!tbv) { + return ParsingResult::kMoreDataNeeded; + } + + ASSIGN_OR_RETURN(AttrsSectionReader attr_reader, + AttrsSectionReader::Create(header_, std::move(*tbv))); + + PerfSession::Builder builder(context_); + while (attr_reader.CanReadNext()) { + PerfFile::AttrsEntry entry; + RETURN_IF_ERROR(attr_reader.ReadNext(entry)); + + if (entry.ids.size % sizeof(uint64_t) != 0) { + return base::ErrStatus("Invalid id section size: %" PRIu64, + entry.ids.size); } - reader_.Skip<perf_event_header>(); - uint64_t record_offset = reader_.current_file_offset(); - uint64_t record_size = ev_header.size - sizeof(perf_event_header); - - switch (ev_header.type) { - case PERF_RECORD_SAMPLE: { - TraceBlobView tbv = reader_.PeekTraceBlobView(record_size); - auto sample_status = tracker_->ParseSample(reader_); - if (!sample_status.ok()) { - continue; - } - PerfDataTracker::PerfSample sample = *sample_status; - if (!ValidateSample(*sample_status)) { - continue; - } - context_->sorter->PushPerfRecord( - static_cast<int64_t>(*sample_status->ts), std::move(tbv)); - break; - } - case PERF_RECORD_MMAP2: { - PERFETTO_CHECK(ev_header.size >= - sizeof(PerfDataTracker::Mmap2Record::Numeric)); - auto record = ParseMmap2Record(record_size); - RETURN_IF_ERROR(record.status()); - record->cpu_mode = GetCpuMode(ev_header); - tracker_->PushMmap2Record(*record); - break; - } - default: - break; + tbv = buffer_.SliceOff(entry.ids.offset, entry.ids.size); + if (!tbv) { + return ParsingResult::kMoreDataNeeded; } - reader_.Skip((record_offset + record_size) - reader_.current_file_offset()); + std::vector<uint64_t> ids; + ids.resize(entry.ids.size / sizeof(uint64_t)); + PERFETTO_CHECK(Reader(std::move(*tbv)).ReadVector(ids)); + + builder.AddAttrAndIds(entry.attr, std::move(ids)); } - return base::OkStatus(); + ASSIGN_OR_RETURN(perf_session_, builder.Build()); + parsing_state_ = ParsingState::kSeekRecords; + return ParsingResult::kSuccess; } base::StatusOr<PerfDataTokenizer::ParsingResult> -PerfDataTokenizer::ParseHeader() { - if (!reader_.CanReadSize(sizeof(PerfHeader))) { - return ParsingResult::NoSpace; +PerfDataTokenizer::SeekRecords() { + if (!buffer_.PopFrontUntil(header_.data.offset)) { + return ParsingResult::kMoreDataNeeded; } - reader_.Read(header_); - PERFETTO_CHECK(header_.size == sizeof(PerfHeader)); - if (header_.attr_size != - sizeof(perf_event_attr) + sizeof(PerfDataTracker::PerfFileSection)) { - return base::ErrStatus( - "Unsupported: perf.data collected with a different ABI version of " - "perf_event_attr."); + parsing_state_ = ParsingState::kParseRecords; + return ParsingResult::kSuccess; +} + +base::StatusOr<PerfDataTokenizer::ParsingResult> +PerfDataTokenizer::ParseRecords() { + while (buffer_.file_offset() < header_.data.end()) { + Record record; + + if (auto res = ParseRecord(record); + !res.ok() || *res != ParsingResult::kSuccess) { + return res; + } + + if (!PushRecord(std::move(record))) { + context_->storage->IncrementStats(stats::perf_record_skipped); + } } - if (header_.attrs.offset > header_.data.offset) { - return base::ErrStatus( - "Can only import files where samples are located after the metadata."); + parsing_state_ = ParsingState::kParseFeatureSections; + return ParsingResult::kSuccess; +} + +base::StatusOr<PerfDataTokenizer::ParsingResult> PerfDataTokenizer::ParseRecord( + Record& record) { + record.session = perf_session_; + std::optional<TraceBlobView> tbv = + buffer_.SliceOff(buffer_.file_offset(), sizeof(record.header)); + if (!tbv) { + return ParsingResult::kMoreDataNeeded; } + PERFETTO_CHECK(Reader(std::move(*tbv)).Read(record.header)); - if (header_.size == header_.attrs.offset) { - parsing_state_ = ParsingState::Attrs; - } else { - parsing_state_ = ParsingState::AfterHeaderBuffer; + if (record.header.size < sizeof(record.header)) { + return base::ErrStatus("Invalid record size: %" PRIu16, record.header.size); } - return ParsingResult::Success; + + tbv = buffer_.SliceOff(buffer_.file_offset() + sizeof(record.header), + record.header.size - sizeof(record.header)); + if (!tbv) { + return ParsingResult::kMoreDataNeeded; + } + + record.payload = std::move(*tbv); + + base::StatusOr<RefPtr<const PerfEventAttr>> attr = + perf_session_->FindAttrForRecord(record.header, record.payload); + if (!attr.ok()) { + return base::ErrStatus("Unable to determine perf_event_attr for record. %s", + attr.status().c_message()); + } + record.attr = *attr; + + buffer_.PopFrontBytes(record.header.size); + return ParsingResult::kSuccess; } -base::StatusOr<PerfDataTokenizer::ParsingResult> -PerfDataTokenizer::ParseAfterHeaderBuffer() { - if (!reader_.CanAccessFileRange(header_.size, header_.attrs.offset)) { - return ParsingResult::NoSpace; +base::StatusOr<int64_t> PerfDataTokenizer::ToTraceTimestamp( + std::optional<uint64_t> time) { + base::StatusOr<int64_t> trace_ts = + time.has_value() + ? context_->clock_tracker->ToTraceTime( + protos::pbzero::ClockSnapshot::Clock::MONOTONIC, + static_cast<int64_t>(*time)) + : std::max(latest_timestamp_, context_->sorter->max_timestamp()); + + if (PERFETTO_LIKELY(trace_ts.ok())) { + latest_timestamp_ = std::max(latest_timestamp_, *trace_ts); } - after_header_buffer_.resize( - static_cast<size_t>(header_.attrs.offset - header_.size)); - reader_.ReadVector(after_header_buffer_); - parsing_state_ = ParsingState::Attrs; - return ParsingResult::Success; + + return trace_ts; } -base::StatusOr<PerfDataTokenizer::ParsingResult> -PerfDataTokenizer::ParseAttrs() { - if (!reader_.CanAccessFileRange(header_.attrs.offset, header_.attrs.end())) { - return ParsingResult::NoSpace; +bool PerfDataTokenizer::PushRecord(Record record) { + std::optional<uint64_t> time; + if (!ReadTime(record, time)) { + return false; } - reader_.Skip(header_.attrs.offset - reader_.current_file_offset()); - PerfDataTracker::PerfFileAttr attr; - for (uint64_t i = header_.attrs.offset; i < header_.attrs.end(); - i += header_.attr_size) { - reader_.Read(attr); - PERFETTO_CHECK(attr.ids.size % sizeof(uint64_t) == 0); - ids_start_ = std::min(ids_start_, attr.ids.offset); - ids_end_ = std::max(ids_end_, attr.ids.end()); - attrs_.push_back(attr); + + base::StatusOr<int64_t> trace_ts = ToTraceTimestamp(time); + if (!trace_ts.ok()) { + return false; } - if (ids_start_ == header_.size && ids_end_ <= header_.attrs.offset) { - parsing_state_ = ParsingState::AttrIdsFromBuffer; - } else { - parsing_state_ = ParsingState::AttrIds; + switch (record.header.type) { + case PERF_RECORD_AUXTRACE_INFO: + case PERF_RECORD_AUXTRACE: + case PERF_RECORD_AUX: + break; + default: + context_->sorter->PushPerfRecord(*trace_ts, std::move(record)); + break; } - return ParsingResult::Success; + + return true; } base::StatusOr<PerfDataTokenizer::ParsingResult> -PerfDataTokenizer::ParseAttrIds() { - if (!reader_.CanAccessFileRange(ids_start_, ids_end_)) { - return ParsingResult::NoSpace; +PerfDataTokenizer::ParseFeatureSections() { + PERFETTO_CHECK(buffer_.file_offset() == header_.data.end()); + auto tbv = buffer_.SliceOff(feature_headers_section_.offset, + feature_headers_section_.size); + if (!tbv) { + return ParsingResult::kMoreDataNeeded; } - for (const auto& attr_file : attrs_) { - reader_.Skip(attr_file.ids.offset - reader_.current_file_offset()); - std::vector<uint64_t> ids(static_cast<size_t>(attr_file.ids.size) / - sizeof(uint64_t)); - reader_.ReadVector(ids); - tracker_->PushAttrAndIds({attr_file.attr, std::move(ids)}); + + Reader reader(std::move(*tbv)); + for (auto feature_id : feature_ids_) { + feature_sections_.emplace_back(std::piecewise_construct, + std::forward_as_tuple(feature_id), + std::forward_as_tuple()); + PERFETTO_CHECK(reader.Read(feature_sections_.back().second)); } - tracker_->ComputeCommonSampleType(); - reader_.Skip(header_.data.offset - reader_.current_file_offset()); - parsing_state_ = ParsingState::Records; - return ParsingResult::Success; + std::sort(feature_sections_.begin(), feature_sections_.end(), + [](const std::pair<uint8_t, PerfFile::Section>& lhs, + const std::pair<uint8_t, PerfFile::Section>& rhs) { + return lhs.second.offset > rhs.second.offset; + }); + + buffer_.PopFrontUntil(feature_headers_section_.end()); + parsing_state_ = feature_sections_.empty() ? ParsingState::kDone + : ParsingState::kParseFeatures; + return ParsingResult::kSuccess; } base::StatusOr<PerfDataTokenizer::ParsingResult> -PerfDataTokenizer::ParseAttrIdsFromBuffer() { - // Each attribute points at an array of event ids. In this case, the ids are - // in |after_header_buffer_|, i.e. the file contents between the header and - // the start of the attr section. - for (const auto& attr_file : attrs_) { - size_t num_ids = static_cast<size_t>(attr_file.ids.size / sizeof(uint64_t)); - std::vector<uint64_t> ids(num_ids); - size_t rd_offset = static_cast<size_t>(attr_file.ids.offset - ids_start_); - size_t rd_size = static_cast<size_t>(attr_file.ids.size); - PERFETTO_CHECK(rd_offset + rd_size <= after_header_buffer_.size()); - memcpy(ids.data(), after_header_buffer_.data() + rd_offset, rd_size); - - tracker_->PushAttrAndIds({attr_file.attr, std::move(ids)}); +PerfDataTokenizer::ParseFeatures() { + while (!feature_sections_.empty()) { + const auto feature_id = feature_sections_.back().first; + const auto& section = feature_sections_.back().second; + auto tbv = buffer_.SliceOff(section.offset, section.size); + if (!tbv) { + return ParsingResult::kMoreDataNeeded; + } + + RETURN_IF_ERROR(ParseFeature(feature_id, std::move(*tbv))); + buffer_.PopFrontUntil(section.end()); + feature_sections_.pop_back(); } - after_header_buffer_.clear(); - tracker_->ComputeCommonSampleType(); - reader_.Skip(header_.data.offset - reader_.current_file_offset()); - parsing_state_ = ParsingState::Records; - return ParsingResult::Success; + parsing_state_ = ParsingState::kDone; + return ParsingResult::kSuccess; } -base::StatusOr<PerfDataTracker::Mmap2Record> -PerfDataTokenizer::ParseMmap2Record(uint64_t record_size) { - uint64_t start_offset = reader_.current_file_offset(); - PerfDataTracker::Mmap2Record record; - reader_.Read(record.num); - std::vector<char> filename_buffer( - static_cast<size_t>(record_size) - - sizeof(PerfDataTracker::Mmap2Record::Numeric)); - reader_.ReadVector(filename_buffer); - if (filename_buffer.back() != '\0') { - return base::ErrStatus( - "Invalid MMAP2 record: filename is not null terminated."); - } - record.filename = std::string(filename_buffer.begin(), filename_buffer.end()); - PERFETTO_CHECK(reader_.current_file_offset() == start_offset + record_size); - return record; -} +base::Status PerfDataTokenizer::ParseFeature(uint8_t feature_id, + TraceBlobView data) { + switch (feature_id) { + case feature::ID_CMD_LINE: { + ASSIGN_OR_RETURN(std::vector<std::string> args, + feature::ParseCmdline(std::move(data))); + perf_session_->SetCmdline(args); + return base::OkStatus(); + } -bool PerfDataTokenizer::ValidateSample( - const PerfDataTracker::PerfSample& sample) { - if (!sample.cpu.has_value() || !sample.ts.has_value() || - sample.callchain.empty() || !sample.pid.has_value()) { - context_->storage->IncrementStats(stats::perf_samples_skipped); - return false; + case feature::ID_EVENT_DESC: + return feature::EventDescription::Parse( + std::move(data), [&](feature::EventDescription desc) { + for (auto id : desc.ids) { + perf_session_->SetEventName(id, std::move(desc.event_string)); + } + return base::OkStatus(); + }); + + case feature::ID_BUILD_ID: + return feature::BuildId::Parse( + std::move(data), [&](feature::BuildId build_id) { + perf_session_->AddBuildId( + build_id.pid, std::move(build_id.filename), + BuildId::FromRaw(std::move(build_id.build_id))); + return base::OkStatus(); + }); + + case feature::ID_GROUP_DESC: { + feature::HeaderGroupDesc group_desc; + RETURN_IF_ERROR( + feature::HeaderGroupDesc::Parse(std::move(data), group_desc)); + // TODO(carlscab): Do someting + break; + } + + case feature::ID_SIMPLEPERF_META_INFO: { + feature::SimpleperfMetaInfo meta_info; + RETURN_IF_ERROR( + feature::SimpleperfMetaInfo::Parse(std::move(data), meta_info)); + for (auto it = meta_info.event_type_info.GetIterator(); it; ++it) { + perf_session_->SetEventName(it.key().type, it.key().config, it.value()); + } + break; + } + case feature::ID_SIMPLEPERF_FILE2: { + RETURN_IF_ERROR(feature::ParseSimpleperfFile2( + std::move(data), [&](TraceBlobView blob) { + third_party::simpleperf::proto::pbzero::FileFeature::Decoder file( + blob.data(), blob.length()); + DsoTracker::GetOrCreate(context_).AddSimpleperfFile2(file); + })); + + break; + } + default: + context_->storage->IncrementIndexedStats(stats::perf_features_skipped, + feature_id); } - return true; + + return base::OkStatus(); } void PerfDataTokenizer::NotifyEndOfFile() {} diff --git a/src/trace_processor/importers/perf/perf_data_tokenizer.h b/src/trace_processor/importers/perf/perf_data_tokenizer.h index 7a54088c8..61c13089c 100644 --- a/src/trace_processor/importers/perf/perf_data_tokenizer.h +++ b/src/trace_processor/importers/perf/perf_data_tokenizer.h @@ -18,45 +18,30 @@ #define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_TOKENIZER_H_ #include <stdint.h> +#include <cstdint> +#include <optional> +#include <vector> + +#include "perfetto/base/flat_set.h" #include "perfetto/base/status.h" #include "perfetto/ext/base/status_or.h" -#include "perfetto/ext/base/string_utils.h" +#include "perfetto/trace_processor/ref_counted.h" #include "perfetto/trace_processor/trace_blob_view.h" -#include "src/trace_processor/importers/perf/perf_data_reader.h" -#include "src/trace_processor/importers/perf/perf_data_tracker.h" -#include "src/trace_processor/importers/perf/perf_event.h" - -#include <limits> -#include <map> -#include <string> -#include <vector> - #include "src/trace_processor/importers/common/chunked_trace_reader.h" +#include "src/trace_processor/importers/perf/perf_file.h" +#include "src/trace_processor/importers/perf/perf_session.h" +#include "src/trace_processor/util/file_buffer.h" namespace perfetto { namespace trace_processor { +class TraceProcessorContext; + namespace perf_importer { -using Section = PerfDataTracker::PerfFileSection; +struct Record; class PerfDataTokenizer : public ChunkedTraceReader { public: - struct PerfHeader { - static constexpr char PERF_MAGIC[] = "PERFILE2"; - - char magic[8]; - uint64_t size; - // Size of PerfFileAttr struct and section pointing to ids. - uint64_t attr_size; - Section attrs; - Section data; - Section event_types; - uint64_t flags; - uint64_t flags1[3]; - - uint64_t num_attrs() const { return attrs.size / attr_size; } - }; - explicit PerfDataTokenizer(TraceProcessorContext*); ~PerfDataTokenizer() override; PerfDataTokenizer(const PerfDataTokenizer&) = delete; @@ -68,39 +53,46 @@ class PerfDataTokenizer : public ChunkedTraceReader { private: enum class ParsingState { - Header = 0, - AfterHeaderBuffer = 1, - Attrs = 2, - AttrIds = 3, - AttrIdsFromBuffer = 4, - Records = 5 + kParseHeader, + kParseAttrs, + kSeekRecords, + kParseRecords, + kParseFeatureSections, + kParseFeatures, + kDone, }; - enum class ParsingResult { NoSpace = 0, Success = 1 }; + enum class ParsingResult { kMoreDataNeeded = 0, kSuccess = 1 }; base::StatusOr<ParsingResult> ParseHeader(); - base::StatusOr<ParsingResult> ParseAfterHeaderBuffer(); base::StatusOr<ParsingResult> ParseAttrs(); - base::StatusOr<ParsingResult> ParseAttrIds(); - base::StatusOr<ParsingResult> ParseAttrIdsFromBuffer(); + base::StatusOr<ParsingResult> SeekRecords(); + base::StatusOr<ParsingResult> ParseRecords(); + base::StatusOr<ParsingResult> ParseFeatureSections(); + base::StatusOr<ParsingResult> ParseFeatures(); - base::StatusOr<PerfDataTracker::Mmap2Record> ParseMmap2Record( - uint64_t record_size); + base::StatusOr<PerfDataTokenizer::ParsingResult> ParseRecord(Record& record); + bool PushRecord(Record record); + base::Status ParseFeature(uint8_t feature_id, TraceBlobView payload); - bool ValidateSample(const PerfDataTracker::PerfSample&); + base::StatusOr<int64_t> ToTraceTimestamp(std::optional<uint64_t> time); TraceProcessorContext* context_; - PerfDataTracker* tracker_; - ParsingState parsing_state_ = ParsingState::Header; + ParsingState parsing_state_ = ParsingState::kParseHeader; + + PerfFile::Header header_; + base::FlatSet<uint8_t> feature_ids_; + PerfFile::Section feature_headers_section_; + // Sections for the features present in the perf file sorted by descending + // section offset. This is done so that we can pop from the back as we process + // the sections. + std::vector<std::pair<uint8_t, PerfFile::Section>> feature_sections_; - PerfHeader header_; + RefPtr<PerfSession> perf_session_; - std::vector<PerfDataTracker::PerfFileAttr> attrs_; - uint64_t ids_start_ = std::numeric_limits<uint64_t>::max(); - uint64_t ids_end_ = 0; - std::vector<uint8_t> after_header_buffer_; + util::FileBuffer buffer_; - perf_importer::PerfDataReader reader_; + int64_t latest_timestamp_ = 0; }; } // namespace perf_importer diff --git a/src/trace_processor/importers/perf/perf_data_tracker.cc b/src/trace_processor/importers/perf/perf_data_tracker.cc deleted file mode 100644 index 8b3bc470a..000000000 --- a/src/trace_processor/importers/perf/perf_data_tracker.cc +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/trace_processor/importers/perf/perf_data_tracker.h" - -#include <optional> - -#include "perfetto/base/status.h" -#include "src/trace_processor/importers/common/address_range.h" -#include "src/trace_processor/importers/common/mapping_tracker.h" -#include "src/trace_processor/importers/common/process_tracker.h" -#include "src/trace_processor/storage/stats.h" -#include "src/trace_processor/storage/trace_storage.h" - -#include "protos/perfetto/trace/profiling/profile_packet.pbzero.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { -namespace { - -bool IsInKernel(protos::pbzero::Profiling::CpuMode cpu_mode) { - switch (cpu_mode) { - case protos::pbzero::Profiling::MODE_UNKNOWN: - PERFETTO_CHECK(false); - case protos::pbzero::Profiling::MODE_GUEST_KERNEL: - case protos::pbzero::Profiling::MODE_KERNEL: - return true; - case protos::pbzero::Profiling::MODE_USER: - case protos::pbzero::Profiling::MODE_HYPERVISOR: - case protos::pbzero::Profiling::MODE_GUEST_USER: - return false; - } - PERFETTO_CHECK(false); -} - -CreateMappingParams BuildCreateMappingParams( - PerfDataTracker::Mmap2Record record) { - return {AddressRange::FromStartAndSize(record.num.addr, record.num.len), - record.num.pgoff, - // start_offset: This is the offset into the file where the ELF header - // starts. We assume all file mappings are ELF files an thus this - // offset is 0. - 0, - // load_bias: This can only be read out of the actual ELF file, which - // we do not have here, so we set it to 0. When symbolizing we will - // hopefully have the real load bias and we can compensate there for a - // possible mismatch. - 0, record.filename, std::nullopt}; -} -} // namespace - -PerfDataTracker::~PerfDataTracker() = default; - -uint64_t PerfDataTracker::ComputeCommonSampleType() { - if (attrs_.empty()) { - return 0; - } - common_sample_type_ = std::numeric_limits<uint64_t>::max(); - for (const auto& a : attrs_) { - common_sample_type_ &= a.attr.sample_type; - } - return common_sample_type_; -} - -const perf_event_attr* PerfDataTracker::FindAttrWithId(uint64_t id) const { - for (const auto& attr_and_ids : attrs_) { - if (auto x = - std::find(attr_and_ids.ids.begin(), attr_and_ids.ids.end(), id); - x == attr_and_ids.ids.end()) { - continue; - } - return &attr_and_ids.attr; - } - return nullptr; -} - -void PerfDataTracker::PushMmap2Record(Mmap2Record record) { - if (IsInKernel(record.cpu_mode)) { - context_->mapping_tracker->CreateKernelMemoryMapping( - BuildCreateMappingParams(std::move(record))); - } else { - UniquePid upid = - context_->process_tracker->GetOrCreateProcess(record.num.pid); - context_->mapping_tracker->CreateUserMemoryMapping( - upid, BuildCreateMappingParams(std::move(record))); - } -} - -base::StatusOr<PerfDataTracker::PerfSample> PerfDataTracker::ParseSample( - perfetto::trace_processor::perf_importer::PerfDataReader& reader) { - uint64_t sample_type = common_sample_type(); - PerfDataTracker::PerfSample sample; - - if (sample_type & PERF_SAMPLE_IDENTIFIER) { - reader.ReadOptional(sample.id); - if (auto attr = FindAttrWithId(*sample.id); attr) { - sample_type = attr->sample_type; - } else { - return base::ErrStatus("No attr for sample_id"); - } - } - - if (sample_type & PERF_SAMPLE_IP) { - reader.Skip<uint64_t>(); - } - - if (sample_type & PERF_SAMPLE_TID) { - reader.ReadOptional(sample.pid); - reader.ReadOptional(sample.tid); - } - - if (sample_type & PERF_SAMPLE_TIME) { - reader.ReadOptional(sample.ts); - } - - // Ignored. Checked because we need to access later parts of sample. - if (sample_type & PERF_SAMPLE_ADDR) { - reader.Skip<uint64_t>(); - } - - // The same value as PERF_SAMPLE_IDENTIFIER, so should be ignored. - if (sample_type & PERF_SAMPLE_ID) { - reader.Skip<uint64_t>(); - } - - // Ignored. Checked because we need to access later parts of sample. - if (sample_type & PERF_SAMPLE_STREAM_ID) { - reader.Skip<uint64_t>(); - } - - if (sample_type & PERF_SAMPLE_CPU) { - reader.ReadOptional(sample.cpu); - // Ignore next uint32_t res. - reader.Skip<uint32_t>(); - } - - // Ignored. Checked because we need to access later parts of sample. - if (sample_type & PERF_SAMPLE_PERIOD) { - reader.Skip<uint64_t>(); - } - - // Ignored. - // TODO(mayzner): Implement. - if (sample_type & PERF_SAMPLE_READ) { - context_->storage->IncrementStats(stats::perf_samples_skipped); - return base::ErrStatus("PERF_SAMPLE_READ is not supported"); - } - - if (sample_type & PERF_SAMPLE_CALLCHAIN) { - uint64_t vec_size; - reader.Read(vec_size); - - sample.callchain.resize(static_cast<size_t>(vec_size)); - reader.ReadVector(sample.callchain); - } - - return sample; -} - -PerfDataTracker* PerfDataTracker::GetOrCreate(TraceProcessorContext* context) { - if (!context->perf_data_tracker) { - context->perf_data_tracker.reset(new PerfDataTracker(context)); - } - return static_cast<PerfDataTracker*>(context->perf_data_tracker.get()); -} -} // namespace perf_importer -} // namespace trace_processor -} // namespace perfetto diff --git a/src/trace_processor/importers/perf/perf_data_tracker.h b/src/trace_processor/importers/perf/perf_data_tracker.h deleted file mode 100644 index c96b08d39..000000000 --- a/src/trace_processor/importers/perf/perf_data_tracker.h +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (C) 2022 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_TRACKER_H_ -#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_TRACKER_H_ - -#include <cstdint> -#include <string> -#include <vector> -#include "perfetto/base/logging.h" -#include "perfetto/base/status.h" -#include "perfetto/ext/base/flat_hash_map.h" -#include "perfetto/ext/base/status_or.h" -#include "perfetto/ext/base/string_utils.h" -#include "protos/perfetto/trace/profiling/profile_packet.pbzero.h" -#include "src/trace_processor/importers/perf/perf_data_reader.h" -#include "src/trace_processor/importers/perf/perf_event.h" -#include "src/trace_processor/storage/trace_storage.h" -#include "src/trace_processor/tables/profiler_tables_py.h" -#include "src/trace_processor/types/destructible.h" -#include "src/trace_processor/types/trace_processor_context.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { - -using MappingTable = tables::StackProfileMappingTable; - -class PerfDataTracker : public Destructible { - public: - struct PerfFileSection { - uint64_t offset; - uint64_t size; - - uint64_t end() const { return offset + size; } - }; - struct PerfFileAttr { - perf_event_attr attr; - PerfFileSection ids; - }; - struct AttrAndIds { - perf_event_attr attr; - std::vector<uint64_t> ids; - }; - struct PerfSample { - std::optional<uint64_t> id = 0; - std::optional<uint32_t> pid = 0; - std::optional<uint32_t> tid = 0; - std::optional<uint64_t> ts = 0; - std::optional<uint32_t> cpu = 0; - std::vector<uint64_t> callchain; - }; - struct Mmap2Record { - struct Numeric { - uint32_t pid; - uint32_t tid; - uint64_t addr; - uint64_t len; - uint64_t pgoff; - uint32_t maj; - uint32_t min; - uint64_t ino; - uint64_t ino_generation; - uint32_t prot; - uint32_t flags; - }; - protos::pbzero::Profiling::CpuMode cpu_mode; - Numeric num; - std::string filename; - }; - - PerfDataTracker(const PerfDataTracker&) = delete; - PerfDataTracker& operator=(const PerfDataTracker&) = delete; - explicit PerfDataTracker(TraceProcessorContext* context) - : context_(context) {} - ~PerfDataTracker() override; - static PerfDataTracker* GetOrCreate(TraceProcessorContext* context); - - uint64_t ComputeCommonSampleType(); - - void PushAttrAndIds(AttrAndIds data) { attrs_.push_back(std::move(data)); } - - void PushMmap2Record(Mmap2Record record); - - uint64_t common_sample_type() { return common_sample_type_; } - - base::StatusOr<PerfSample> ParseSample( - perfetto::trace_processor::perf_importer::PerfDataReader&); - - private: - const perf_event_attr* FindAttrWithId(uint64_t id) const; - TraceProcessorContext* context_; - std::vector<AttrAndIds> attrs_; - - uint64_t common_sample_type_; -}; -} // namespace perf_importer -} // namespace trace_processor -} // namespace perfetto - -#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_DATA_TRACKER_H_ diff --git a/src/trace_processor/importers/perf/perf_data_tracker_unittest.cc b/src/trace_processor/importers/perf/perf_data_tracker_unittest.cc deleted file mode 100644 index 923473f3a..000000000 --- a/src/trace_processor/importers/perf/perf_data_tracker_unittest.cc +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/trace_processor/importers/perf/perf_data_tracker.h" - -#include <stddef.h> -#include <cstring> -#include <memory> -#include <vector> - -#include "perfetto/base/build_config.h" -#include "protos/perfetto/trace/profiling/profile_packet.pbzero.h" -#include "src/trace_processor/importers/common/address_range.h" -#include "src/trace_processor/importers/common/mapping_tracker.h" -#include "src/trace_processor/importers/common/process_tracker.h" -#include "src/trace_processor/importers/common/stack_profile_tracker.h" -#include "src/trace_processor/importers/perf/perf_event.h" -#include "test/gtest_and_gmock.h" - -namespace perfetto { -namespace trace_processor { -namespace perf_importer { -namespace { - -class PerfDataTrackerUnittest : public testing::Test { - public: - PerfDataTrackerUnittest() { - context_.storage = std::make_unique<TraceStorage>(); - context_.process_tracker = std::make_unique<ProcessTracker>(&context_); - context_.stack_profile_tracker = - std::make_unique<StackProfileTracker>(&context_); - context_.mapping_tracker = std::make_unique<MappingTracker>(&context_); - } - - protected: - TraceProcessorContext context_; -}; - -TEST_F(PerfDataTrackerUnittest, ComputeCommonSampleType) { - PerfDataTracker* tracker = PerfDataTracker::GetOrCreate(&context_); - - PerfDataTracker::AttrAndIds attr_and_ids; - attr_and_ids.attr.sample_type = - PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_CPU | PERF_SAMPLE_TIME; - tracker->PushAttrAndIds(attr_and_ids); - - attr_and_ids.attr.sample_type = PERF_SAMPLE_ADDR | PERF_SAMPLE_CPU; - tracker->PushAttrAndIds(attr_and_ids); - - tracker->ComputeCommonSampleType(); - EXPECT_TRUE(tracker->common_sample_type() & PERF_SAMPLE_CPU); - EXPECT_FALSE(tracker->common_sample_type() & PERF_SAMPLE_CALLCHAIN); -} - -TEST_F(PerfDataTrackerUnittest, FindMapping) { - PerfDataTracker* tracker = PerfDataTracker::GetOrCreate(&context_); - - PerfDataTracker::Mmap2Record rec; - rec.cpu_mode = protos::pbzero::Profiling::MODE_USER; - rec.filename = "file1"; - rec.num.addr = 1000; - rec.num.len = 100; - rec.num.pid = 1; - rec.cpu_mode = protos::pbzero::Profiling::MODE_USER; - tracker->PushMmap2Record(rec); - - rec.num.addr = 2000; - tracker->PushMmap2Record(rec); - - rec.num.addr = 3000; - tracker->PushMmap2Record(rec); - - UserMemoryMapping* mapping = - context_.mapping_tracker->FindUserMappingForAddress( - context_.process_tracker->GetOrCreateProcess(1), 2050); - ASSERT_NE(mapping, nullptr); - EXPECT_EQ(mapping->memory_range().start(), 2000u); - EXPECT_EQ(mapping->memory_range().end(), 2100u); -} - -TEST_F(PerfDataTrackerUnittest, FindMappingFalse) { - PerfDataTracker* tracker = PerfDataTracker::GetOrCreate(&context_); - - PerfDataTracker::Mmap2Record rec; - rec.cpu_mode = protos::pbzero::Profiling::MODE_USER; - rec.filename = "file1"; - rec.num.addr = 1000; - rec.num.len = 100; - rec.num.pid = 1; - rec.cpu_mode = protos::pbzero::Profiling::MODE_USER; - tracker->PushMmap2Record(rec); - - UserMemoryMapping* mapping = - context_.mapping_tracker->FindUserMappingForAddress( - context_.process_tracker->GetOrCreateProcess(2), 2050); - EXPECT_EQ(mapping, nullptr); -} - -TEST_F(PerfDataTrackerUnittest, ParseSampleTrivial) { - PerfDataTracker* tracker = PerfDataTracker::GetOrCreate(&context_); - - PerfDataTracker::AttrAndIds attr_and_ids; - attr_and_ids.attr.sample_type = PERF_SAMPLE_TIME; - tracker->PushAttrAndIds(attr_and_ids); - tracker->ComputeCommonSampleType(); - - uint64_t ts = 100; - - TraceBlob blob = - TraceBlob::CopyFrom(static_cast<const void*>(&ts), sizeof(uint64_t)); - PerfDataReader reader(TraceBlobView(std::move(blob))); - - auto parsed_sample = tracker->ParseSample(reader); - EXPECT_TRUE(parsed_sample.ok()); - EXPECT_EQ(parsed_sample->ts, 100u); -} - -TEST_F(PerfDataTrackerUnittest, ParseSampleCallchain) { - PerfDataTracker* tracker = PerfDataTracker::GetOrCreate(&context_); - - PerfDataTracker::AttrAndIds attr_and_ids; - attr_and_ids.attr.sample_type = PERF_SAMPLE_CALLCHAIN; - tracker->PushAttrAndIds(attr_and_ids); - tracker->ComputeCommonSampleType(); - - struct Sample { - uint64_t callchain_size; /* if PERF_SAMPLE_CALLCHAIN */ - std::vector<uint64_t> callchain; /* if PERF_SAMPLE_CALLCHAIN */ - }; - - Sample sample; - sample.callchain_size = 3; - sample.callchain = std::vector<uint64_t>{1, 2, 3}; - - TraceBlob blob = TraceBlob::Allocate(4 * sizeof(uint64_t)); - memcpy(blob.data(), &sample.callchain_size, sizeof(uint64_t)); - memcpy(blob.data() + sizeof(uint64_t), sample.callchain.data(), - sizeof(uint64_t) * 3); - PerfDataReader reader(TraceBlobView(std::move(blob))); - - auto parsed_sample = tracker->ParseSample(reader); - EXPECT_TRUE(parsed_sample.ok()); - EXPECT_EQ(parsed_sample->callchain.size(), 3u); -} - -TEST_F(PerfDataTrackerUnittest, ParseSampleWithoutId) { - PerfDataTracker* tracker = PerfDataTracker::GetOrCreate(&context_); - - PerfDataTracker::AttrAndIds attr_and_ids; - attr_and_ids.attr.sample_type = PERF_SAMPLE_TID | PERF_SAMPLE_TIME | - PERF_SAMPLE_CPU | PERF_SAMPLE_CALLCHAIN; - tracker->PushAttrAndIds(attr_and_ids); - tracker->ComputeCommonSampleType(); - - struct Sample { - uint32_t pid; /* if PERF_SAMPLE_TID */ - uint32_t tid; /* if PERF_SAMPLE_TID */ - uint64_t ts; /* if PERF_SAMPLE_TIME */ - uint32_t cpu; /* if PERF_SAMPLE_CPU */ - uint32_t res_ignore; /* if PERF_SAMPLE_CPU */ - uint64_t callchain_size; /* if PERF_SAMPLE_CALLCHAIN */ - }; - - Sample sample; - sample.pid = 2; - sample.ts = 100; - sample.cpu = 1; - sample.callchain_size = 3; - std::vector<uint64_t> callchain{1, 2, 3}; - - TraceBlob blob = TraceBlob::Allocate(sizeof(Sample) + sizeof(uint64_t) * 3); - memcpy(blob.data(), &sample, sizeof(Sample)); - memcpy(blob.data() + sizeof(Sample), callchain.data(), sizeof(uint64_t) * 3); - - PerfDataReader reader(TraceBlobView(std::move(blob))); - EXPECT_TRUE(reader.CanReadSize(sizeof(Sample))); - - auto parsed_sample = tracker->ParseSample(reader); - EXPECT_TRUE(parsed_sample.ok()); - EXPECT_EQ(parsed_sample->callchain.size(), 3u); - EXPECT_EQ(sample.ts, parsed_sample->ts); -} - -TEST_F(PerfDataTrackerUnittest, ParseSampleWithId) { - PerfDataTracker* tracker = PerfDataTracker::GetOrCreate(&context_); - - PerfDataTracker::AttrAndIds attr_and_ids; - attr_and_ids.attr.sample_type = PERF_SAMPLE_CPU | PERF_SAMPLE_TID | - PERF_SAMPLE_IDENTIFIER | PERF_SAMPLE_ID | - PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_TIME; - attr_and_ids.ids.push_back(10); - tracker->PushAttrAndIds(attr_and_ids); - tracker->ComputeCommonSampleType(); - - struct Sample { - uint64_t identifier; /* if PERF_SAMPLE_IDENTIFIER */ - uint32_t pid; /* if PERF_SAMPLE_TID */ - uint32_t tid; /* if PERF_SAMPLE_TID */ - uint64_t ts; /* if PERF_SAMPLE_TIME */ - uint64_t id; /* if PERF_SAMPLE_ID */ - uint32_t cpu; /* if PERF_SAMPLE_CPU */ - uint32_t res_ignore; /* if PERF_SAMPLE_CPU */ - uint64_t callchain_size; /* if PERF_SAMPLE_CALLCHAIN */ - }; - - Sample sample; - sample.id = 10; - sample.identifier = 10; - sample.cpu = 1; - sample.pid = 2; - sample.ts = 100; - sample.callchain_size = 3; - std::vector<uint64_t> callchain{1, 2, 3}; - - TraceBlob blob = TraceBlob::Allocate(sizeof(Sample) + sizeof(uint64_t) * 3); - memcpy(blob.data(), &sample, sizeof(Sample)); - memcpy(blob.data() + sizeof(Sample), callchain.data(), sizeof(uint64_t) * 3); - - PerfDataReader reader(TraceBlobView(std::move(blob))); - - auto parsed_sample = tracker->ParseSample(reader); - EXPECT_TRUE(parsed_sample.ok()); - EXPECT_EQ(parsed_sample->callchain.size(), 3u); - EXPECT_EQ(100u, parsed_sample->ts); -} - -} // namespace -} // namespace perf_importer -} // namespace trace_processor -} // namespace perfetto diff --git a/src/trace_processor/importers/perf/perf_event.h b/src/trace_processor/importers/perf/perf_event.h index 4763e23f7..58077be2e 100644 --- a/src/trace_processor/importers/perf/perf_event.h +++ b/src/trace_processor/importers/perf/perf_event.h @@ -238,6 +238,7 @@ enum perf_record_misc { PERF_RECORD_MISC_GUEST_USER = 5, PERF_RECORD_MISC_MMAP_BUILD_ID = 1U << 14, + PERF_RECORD_MISC_EXT_RESERVED = 1U << 15, }; enum perf_event_read_format { @@ -250,7 +251,7 @@ enum perf_event_read_format { PERF_FORMAT_MAX = 1U << 5, /* non-ABI */ }; -enum perf_callchain_context { +enum perf_callchain_context : uint64_t { PERF_CONTEXT_HV = static_cast<uint64_t>(-32), PERF_CONTEXT_KERNEL = static_cast<uint64_t>(-128), PERF_CONTEXT_USER = static_cast<uint64_t>(-512), diff --git a/src/trace_processor/importers/perf/perf_event_attr.cc b/src/trace_processor/importers/perf/perf_event_attr.cc index 1abf8d0aa..b6c3edce7 100644 --- a/src/trace_processor/importers/perf/perf_event_attr.cc +++ b/src/trace_processor/importers/perf/perf_event_attr.cc @@ -21,7 +21,12 @@ #include <cstring> #include <optional> +#include "perfetto/ext/base/string_view.h" +#include "src/trace_processor/importers/perf/perf_counter.h" #include "src/trace_processor/importers/perf/perf_event.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/profiler_tables_py.h" +#include "src/trace_processor/types/trace_processor_context.h" namespace perfetto::trace_processor::perf_importer { @@ -90,11 +95,41 @@ std::optional<size_t> IdOffsetFromEndOfNonSampleRecord( } } // namespace -PerfEventAttr::PerfEventAttr(perf_event_attr attr) - : attr_(std::move(attr)), +PerfEventAttr::PerfEventAttr(TraceProcessorContext* context, + tables::PerfSessionTable::Id perf_session_id, + perf_event_attr attr) + : context_(context), + perf_session_id_(perf_session_id), + attr_(std::move(attr)), time_offset_from_start_(TimeOffsetFromStartOfSampleRecord(attr_)), time_offset_from_end_(TimeOffsetFromEndOfNonSampleRecord(attr_)), id_offset_from_start_(IdOffsetFromStartOfSampleRecord(attr_)), id_offset_from_end_(IdOffsetFromEndOfNonSampleRecord(attr_)) {} +PerfEventAttr::~PerfEventAttr() = default; + +PerfCounter& PerfEventAttr::GetOrCreateCounter(uint32_t cpu) const { + auto it = counters_.find(cpu); + if (it == counters_.end()) { + it = counters_.emplace(cpu, CreateCounter(cpu)).first; + } + return it->second; +} + +PerfCounter PerfEventAttr::CreateCounter(uint32_t cpu) const { + tables::PerfCounterTrackTable::Row row; + row.name = context_->storage->InternString(base::StringView(event_name_)); + row.unit = context_->storage->InternString(base::StringView("")); + row.description = context_->storage->InternString(base::StringView("")); + row.perf_session_id = perf_session_id_; + row.cpu = cpu; + row.is_timebase = is_timebase(); + const auto counter_track_ref = + context_->storage->mutable_perf_counter_track_table() + ->Insert(std::move(row)) + .row_reference; + return PerfCounter(context_->storage->mutable_counter_table(), + std::move(counter_track_ref)); +} + } // namespace perfetto::trace_processor::perf_importer diff --git a/src/trace_processor/importers/perf/perf_event_attr.h b/src/trace_processor/importers/perf/perf_event_attr.h index 4f219aadf..57a0c769b 100644 --- a/src/trace_processor/importers/perf/perf_event_attr.h +++ b/src/trace_processor/importers/perf/perf_event_attr.h @@ -21,17 +21,28 @@ #include <cstddef> #include <cstdint> #include <optional> +#include <unordered_map> -#include "perfetto/ext/base/string_view.h" #include "perfetto/trace_processor/ref_counted.h" +#include "src/trace_processor/importers/perf/perf_counter.h" #include "src/trace_processor/importers/perf/perf_event.h" +#include "src/trace_processor/tables/profiler_tables_py.h" -namespace perfetto::trace_processor::perf_importer { +namespace perfetto::trace_processor { + +class TraceProcessorContext; + +namespace perf_importer { // Wrapper around a `perf_event_attr` object that add some helper methods. class PerfEventAttr : public RefCounted { public: - explicit PerfEventAttr(perf_event_attr attr); + PerfEventAttr(TraceProcessorContext* context, + tables::PerfSessionTable::Id perf_session_id_, + perf_event_attr attr); + ~PerfEventAttr(); + uint32_t type() const { return attr_.type; } + uint64_t config() const { return attr_.config; } uint64_t sample_type() const { return attr_.sample_type; } uint64_t read_format() const { return attr_.read_format; } bool sample_id_all() const { return !!attr_.sample_id_all; } @@ -48,12 +59,6 @@ class PerfEventAttr : public RefCounted { return attr_.freq ? std::make_optional(attr_.sample_freq) : std::nullopt; } - bool is_timebase() const { - // This is what simpleperf uses for events that are not supposed to sample - // TODO(b/334978369): Determine if there is a better way to figure this out. - return attr_.sample_period < (1ull << 62); - } - // Offset from the end of a record's payload to the time filed (if present). // To be used with non `PERF_RECORD_SAMPLE` records std::optional<size_t> time_offset_from_end() const { @@ -81,14 +86,33 @@ class PerfEventAttr : public RefCounted { return id_offset_from_end_; } + void set_event_name(std::string event_name) { + event_name_ = std::move(event_name); + } + + PerfCounter& GetOrCreateCounter(uint32_t cpu) const; + private: + bool is_timebase() const { + // This is what simpleperf uses for events that are not supposed to sample + // TODO(b/334978369): Determine if there is a better way to figure this out. + return attr_.sample_period < (1ull << 62); + } + + PerfCounter CreateCounter(uint32_t cpu) const; + + TraceProcessorContext* const context_; + tables::PerfSessionTable::Id perf_session_id_; perf_event_attr attr_; std::optional<size_t> time_offset_from_start_; std::optional<size_t> time_offset_from_end_; std::optional<size_t> id_offset_from_start_; std::optional<size_t> id_offset_from_end_; + mutable std::unordered_map<uint32_t, PerfCounter> counters_; + std::string event_name_; }; -} // namespace perfetto::trace_processor::perf_importer +} // namespace perf_importer +} // namespace perfetto::trace_processor #endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_EVENT_ATTR_H_ diff --git a/src/trace_processor/importers/perf/perf_session.cc b/src/trace_processor/importers/perf/perf_session.cc index 86d983d0b..9923fcbad 100644 --- a/src/trace_processor/importers/perf/perf_session.cc +++ b/src/trace_processor/importers/perf/perf_session.cc @@ -20,17 +20,24 @@ #include <cstddef> #include <cstdint> #include <cstring> +#include <limits> #include <optional> +#include <string> #include <vector> #include "perfetto/base/logging.h" #include "perfetto/base/status.h" #include "perfetto/ext/base/flat_hash_map.h" #include "perfetto/ext/base/status_or.h" +#include "perfetto/ext/base/string_utils.h" +#include "perfetto/ext/base/string_view.h" #include "perfetto/trace_processor/ref_counted.h" #include "src/trace_processor/importers/perf/perf_event.h" #include "src/trace_processor/importers/perf/perf_event_attr.h" #include "src/trace_processor/importers/perf/reader.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/types/trace_processor_context.h" +#include "src/trace_processor/util/build_id.h" namespace perfetto::trace_processor::perf_importer { namespace { @@ -46,11 +53,16 @@ base::StatusOr<RefPtr<PerfSession>> PerfSession::Builder::Build() { return base::ErrStatus("No perf_event_attr"); } - const PerfEventAttr base_attr(attr_with_ids_[0].attr); + auto perf_session_id = + context_->storage->mutable_perf_session_table()->Insert({}).id; + + const PerfEventAttr base_attr(context_, perf_session_id, + attr_with_ids_[0].attr); base::FlatHashMap<uint64_t, RefPtr<PerfEventAttr>> attrs_by_id; for (const auto& entry : attr_with_ids_) { - RefPtr<PerfEventAttr> attr(new PerfEventAttr(entry.attr)); + RefPtr<PerfEventAttr> attr( + new PerfEventAttr(context_, perf_session_id, entry.attr)); if (base_attr.sample_id_all() != attr->sample_id_all()) { return base::ErrStatus( "perf_event_attr with different sample_id_all values"); @@ -75,8 +87,9 @@ base::StatusOr<RefPtr<PerfSession>> PerfSession::Builder::Build() { return base::ErrStatus("No id offsets for multiple perf_event_attr"); } - return RefPtr<PerfSession>(new PerfSession( - perf_session_id_, std::move(attrs_by_id), attr_with_ids_.size() == 1)); + return RefPtr<PerfSession>(new PerfSession(context_, perf_session_id, + std::move(attrs_by_id), + attr_with_ids_.size() == 1)); } base::StatusOr<RefPtr<const PerfEventAttr>> PerfSession::FindAttrForRecord( @@ -133,4 +146,47 @@ RefPtr<const PerfEventAttr> PerfSession::FindAttrForEventId(uint64_t id) const { return RefPtr<const PerfEventAttr>(it->get()); } +void PerfSession::SetEventName(uint64_t event_id, std::string name) { + auto it = attrs_by_id_.Find(event_id); + if (!it) { + return; + } + (*it)->set_event_name(std::move(name)); +} + +void PerfSession::SetEventName(uint32_t type, + uint64_t config, + const std::string& name) { + for (auto it = attrs_by_id_.GetIterator(); it; ++it) { + if (it.value()->type() == type && it.value()->config() == config) { + it.value()->set_event_name(name); + } + } +} + +void PerfSession::AddBuildId(int32_t pid, + std::string filename, + BuildId build_id) { + build_ids_.Insert({pid, std::move(filename)}, std::move(build_id)); +} + +std::optional<BuildId> PerfSession::LookupBuildId( + uint32_t pid, + const std::string& filename) const { + // -1 is used in BUILD_ID feature to match any pid. + static constexpr int32_t kAnyPid = -1; + auto it = build_ids_.Find({static_cast<int32_t>(pid), filename}); + if (!it) { + it = build_ids_.Find({kAnyPid, filename}); + } + return it ? std::make_optional(*it) : std::nullopt; +} + +void PerfSession::SetCmdline(const std::vector<std::string>& args) { + context_->storage->mutable_perf_session_table() + ->FindById(perf_session_id_) + ->set_cmdline(context_->storage->InternString( + base::StringView(base::Join(args, " ")))); +} + } // namespace perfetto::trace_processor::perf_importer diff --git a/src/trace_processor/importers/perf/perf_session.h b/src/trace_processor/importers/perf/perf_session.h index abea41bec..796969c82 100644 --- a/src/trace_processor/importers/perf/perf_session.h +++ b/src/trace_processor/importers/perf/perf_session.h @@ -21,23 +21,31 @@ #include <cstddef> #include <cstdint> #include <optional> +#include <string> +#include <vector> #include "perfetto/ext/base/flat_hash_map.h" +#include "perfetto/ext/base/hash.h" #include "perfetto/ext/base/status_or.h" #include "perfetto/trace_processor/ref_counted.h" #include "perfetto/trace_processor/trace_blob_view.h" #include "src/trace_processor/importers/perf/perf_event.h" #include "src/trace_processor/importers/perf/perf_event_attr.h" +#include "src/trace_processor/tables/profiler_tables_py.h" +#include "src/trace_processor/util/build_id.h" -namespace perfetto::trace_processor::perf_importer { +namespace perfetto::trace_processor { + +class TraceProcessorContext; + +namespace perf_importer { // Helper to deal with perf_event_attr instances in a perf file. class PerfSession : public RefCounted { public: class Builder { public: - explicit Builder(uint32_t perf_session_id) - : perf_session_id_(perf_session_id) {} + explicit Builder(TraceProcessorContext* context) : context_(context) {} base::StatusOr<RefPtr<PerfSession>> Build(); Builder& AddAttrAndIds(perf_event_attr attr, std::vector<uint64_t> ids) { attr_with_ids_.push_back({std::move(attr), std::move(ids)}); @@ -50,11 +58,13 @@ class PerfSession : public RefCounted { std::vector<uint64_t> ids; }; - uint32_t perf_session_id_; + TraceProcessorContext* const context_; std::vector<PerfEventAttrWithIds> attr_with_ids_; }; - uint32_t perf_session_id() const { return perf_session_id_; } + tables::PerfSessionTable::Id perf_session_id() const { + return perf_session_id_; + } RefPtr<const PerfEventAttr> FindAttrForEventId(uint64_t id) const; @@ -62,11 +72,36 @@ class PerfSession : public RefCounted { const perf_event_header& header, const TraceBlobView& payload) const; + void SetCmdline(const std::vector<std::string>& args); + void SetEventName(uint64_t event_id, std::string name); + void SetEventName(uint32_t type, uint64_t config, const std::string& name); + + void AddBuildId(int32_t pid, std::string filename, BuildId build_id); + std::optional<BuildId> LookupBuildId(uint32_t pid, + const std::string& filename) const; + private: - PerfSession(uint32_t perf_session_id, + struct BuildIdMapKey { + int32_t pid; + std::string filename; + + struct Hasher { + size_t operator()(const BuildIdMapKey& k) const { + return static_cast<size_t>(base::Hasher::Combine(k.pid, k.filename)); + } + }; + + bool operator==(const BuildIdMapKey& o) const { + return pid == o.pid && filename == o.filename; + } + }; + + PerfSession(TraceProcessorContext* context, + tables::PerfSessionTable::Id perf_session_id, base::FlatHashMap<uint64_t, RefPtr<PerfEventAttr>> attrs_by_id, bool has_single_perf_event_attr) - : perf_session_id_(perf_session_id), + : context_(context), + perf_session_id_(perf_session_id), attrs_by_id_(std::move(attrs_by_id)), has_single_perf_event_attr_(has_single_perf_event_attr) {} @@ -74,15 +109,19 @@ class PerfSession : public RefCounted { const TraceBlobView& payload, uint64_t& id) const; - uint32_t perf_session_id_; + TraceProcessorContext* const context_; + tables::PerfSessionTable::Id perf_session_id_; base::FlatHashMap<uint64_t, RefPtr<PerfEventAttr>> attrs_by_id_; // Multiple ids can map to the same perf_event_attr. This member tells us // whether there was only one perf_event_attr (with potentially different ids // associated). This makes the attr lookup given a record trivial and not // dependant no having any id field in the records. bool has_single_perf_event_attr_; + + base::FlatHashMap<BuildIdMapKey, BuildId, BuildIdMapKey::Hasher> build_ids_; }; -} // namespace perfetto::trace_processor::perf_importer +} // namespace perf_importer +} // namespace perfetto::trace_processor #endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_PERF_SESSION_H_ diff --git a/src/trace_processor/importers/perf/perf_session_unittest.cc b/src/trace_processor/importers/perf/perf_session_unittest.cc index 0ab85f23a..2b15af862 100644 --- a/src/trace_processor/importers/perf/perf_session_unittest.cc +++ b/src/trace_processor/importers/perf/perf_session_unittest.cc @@ -23,6 +23,8 @@ #include "perfetto/trace_processor/trace_blob.h" #include "perfetto/trace_processor/trace_blob_view.h" #include "src/trace_processor/importers/perf/perf_event.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/types/trace_processor_context.h" #include "test/gtest_and_gmock.h" namespace perfetto::trace_processor::perf_importer { @@ -41,12 +43,16 @@ MATCHER_P(IsOkAndHolds, matcher, "") { } TEST(PerfSessionTest, NoAttrBuildFails) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); EXPECT_FALSE(builder.Build().ok()); } TEST(PerfSessionTest, OneAttrAndNoIdBuildSucceeds) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = false; attr.sample_type = PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_CPU | PERF_SAMPLE_TIME; @@ -61,7 +67,9 @@ TEST(PerfSessionTest, OneAttrAndNoIdBuildSucceeds) { } TEST(PerfSessionTest, MultipleAttrsAndNoIdBuildFails) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_CPU | PERF_SAMPLE_TIME; @@ -71,7 +79,9 @@ TEST(PerfSessionTest, MultipleAttrsAndNoIdBuildFails) { } TEST(PerfSessionTest, MultipleIdsSameAttrAndNoIdCanExtractAttrFromRecord) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_IP | PERF_SAMPLE_CPU | PERF_SAMPLE_TIME; @@ -95,7 +105,9 @@ TEST(PerfSessionTest, MultipleIdsSameAttrAndNoIdCanExtractAttrFromRecord) { } TEST(PerfSessionTest, NoCommonSampleIdAllBuildFails) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_IDENTIFIER; @@ -111,7 +123,9 @@ TEST(PerfSessionTest, NoCommonSampleIdAllBuildFails) { } TEST(PerfSessionTest, NoCommonOffsetForSampleBuildFails) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_IP | PERF_SAMPLE_ID; @@ -122,7 +136,9 @@ TEST(PerfSessionTest, NoCommonOffsetForSampleBuildFails) { } TEST(PerfSessionTest, NoCommonOffsetForNonSampleBuildFails) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_ID | PERF_SAMPLE_TID; @@ -138,7 +154,9 @@ TEST(PerfSessionTest, NoCommonOffsetForNonSampleBuildFails) { } TEST(PerfSessionTest, NoCommonOffsetForNonSampleAndNoSampleIdAllBuildSucceeds) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = false; attr.sample_type = PERF_SAMPLE_IDENTIFIER | PERF_SAMPLE_TID; @@ -149,7 +167,9 @@ TEST(PerfSessionTest, NoCommonOffsetForNonSampleAndNoSampleIdAllBuildSucceeds) { } TEST(PerfSessionTest, MultiplesessionBuildSucceeds) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_IP | PERF_SAMPLE_ID; @@ -159,7 +179,9 @@ TEST(PerfSessionTest, MultiplesessionBuildSucceeds) { } TEST(PerfSessionTest, FindAttrInRecordWithId) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_IP | PERF_SAMPLE_ID; @@ -194,7 +216,9 @@ TEST(PerfSessionTest, FindAttrInRecordWithId) { } TEST(PerfSessionTest, FindAttrInRecordWithIdentifier) { - PerfSession::Builder builder(0); + TraceProcessorContext context; + context.storage.reset(new TraceStorage()); + PerfSession::Builder builder(&context); perf_event_attr attr; attr.sample_id_all = true; attr.sample_type = PERF_SAMPLE_IDENTIFIER | PERF_SAMPLE_IP; diff --git a/src/trace_processor/importers/perf/reader.h b/src/trace_processor/importers/perf/reader.h index faf31a215..25d4f4c3e 100644 --- a/src/trace_processor/importers/perf/reader.h +++ b/src/trace_processor/importers/perf/reader.h @@ -26,7 +26,9 @@ #include <type_traits> #include <vector> +#include "perfetto/ext/base/string_view.h" #include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/importers/perf/perf_event.h" namespace perfetto::trace_processor::perf_importer { @@ -45,6 +47,50 @@ class Reader { // methods are called. size_t size_left() const { return static_cast<size_t>(end_ - current_); } + bool ReadStringView(base::StringView& str, size_t size) { + if (size_left() < size) { + return false; + } + str = base::StringView(reinterpret_cast<const char*>(current_), size); + current_ += size; + return true; + } + + bool ReadPerfEventAttr(perf_event_attr& attr, size_t attr_size) { + const size_t bytes_to_read = std::min(attr_size, sizeof(attr)); + const size_t bytes_to_skip = attr_size - bytes_to_read; + static_assert(std::has_unique_object_representations_v<perf_event_attr>); + + if (size_left() < bytes_to_read + bytes_to_skip) { + return false; + } + + memset(&attr, 0, sizeof(attr)); + + return Read(&attr, bytes_to_read) && Skip(bytes_to_skip); + } + + bool ReadBlob(TraceBlobView& blob, uint32_t size) { + if (size_left() < size) { + return false; + } + blob = TraceBlobView(buffer_, + static_cast<size_t>(current_ - buffer_->data()), size); + current_ += size; + return true; + } + + bool ReadStringUntilEndOrNull(std::string& out) { + const uint8_t* ptr = current_; + while (ptr != end_ && *ptr != 0) { + ++ptr; + } + out = std::string(reinterpret_cast<const char*>(current_), + static_cast<size_t>(ptr - current_)); + current_ = ptr; + return true; + } + template <typename T> bool Read(T& obj) { static_assert(std::has_unique_object_representations_v<T>); diff --git a/src/trace_processor/importers/perf/record_parser.cc b/src/trace_processor/importers/perf/record_parser.cc new file mode 100644 index 000000000..c36d3dbe6 --- /dev/null +++ b/src/trace_processor/importers/perf/record_parser.cc @@ -0,0 +1,319 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/perf/record_parser.h" + +#include <cstdint> +#include <optional> +#include <string> +#include <vector> + +#include "perfetto/base/logging.h" +#include "perfetto/base/status.h" +#include "perfetto/ext/base/string_view.h" +#include "perfetto/public/compiler.h" +#include "perfetto/trace_processor/ref_counted.h" +#include "src/trace_processor/importers/common/mapping_tracker.h" +#include "src/trace_processor/importers/common/process_tracker.h" +#include "src/trace_processor/importers/common/stack_profile_tracker.h" +#include "src/trace_processor/importers/perf/perf_counter.h" +#include "src/trace_processor/importers/perf/perf_event.h" +#include "src/trace_processor/importers/perf/perf_event_attr.h" +#include "src/trace_processor/importers/perf/reader.h" +#include "src/trace_processor/importers/perf/record.h" +#include "src/trace_processor/importers/perf/sample.h" +#include "src/trace_processor/importers/proto/perf_sample_tracker.h" +#include "src/trace_processor/importers/proto/profile_packet_utils.h" +#include "src/trace_processor/storage/stats.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/profiler_tables_py.h" +#include "src/trace_processor/util/build_id.h" +#include "src/trace_processor/util/status_macros.h" + +namespace perfetto { +namespace trace_processor { +namespace perf_importer { +namespace { + +CreateMappingParams BuildCreateMappingParams( + const CommonMmapRecordFields& fields, + std::string filename, + std::optional<BuildId> build_id) { + return {AddressRange::FromStartAndSize(fields.addr, fields.len), fields.pgoff, + // start_offset: This is the offset into the file where the ELF header + // starts. We assume all file mappings are ELF files an thus this + // offset is 0. + 0, + // load_bias: This can only be read out of the actual ELF file, which + // we do not have here, so we set it to 0. When symbolizing we will + // hopefully have the real load bias and we can compensate there for a + // possible mismatch. + 0, std::move(filename), std::move(build_id)}; +} + +bool IsInKernel(protos::pbzero::Profiling::CpuMode cpu_mode) { + switch (cpu_mode) { + case protos::pbzero::Profiling::MODE_UNKNOWN: + PERFETTO_FATAL("Unknown CPU mode"); + case protos::pbzero::Profiling::MODE_GUEST_KERNEL: + case protos::pbzero::Profiling::MODE_KERNEL: + return true; + case protos::pbzero::Profiling::MODE_USER: + case protos::pbzero::Profiling::MODE_HYPERVISOR: + case protos::pbzero::Profiling::MODE_GUEST_USER: + return false; + } + PERFETTO_FATAL("For GCC."); +} + +} // namespace + +using FramesTable = tables::StackProfileFrameTable; +using CallsitesTable = tables::StackProfileCallsiteTable; + +RecordParser::RecordParser(TraceProcessorContext* context) + : context_(context) {} + +RecordParser::~RecordParser() = default; + +void RecordParser::ParsePerfRecord(int64_t ts, Record record) { + if (base::Status status = ParseRecord(ts, std::move(record)); !status.ok()) { + context_->storage->IncrementStats(record.header.type == PERF_RECORD_SAMPLE + ? stats::perf_samples_skipped + : stats::perf_record_skipped); + } +} + +base::Status RecordParser::ParseRecord(int64_t ts, Record record) { + switch (record.header.type) { + case PERF_RECORD_COMM: + return ParseComm(std::move(record)); + + case PERF_RECORD_SAMPLE: + return ParseSample(ts, std::move(record)); + + case PERF_RECORD_MMAP: + return ParseMmap(std::move(record)); + + case PERF_RECORD_MMAP2: + return ParseMmap2(std::move(record)); + + case PERF_RECORD_AUX: + case PERF_RECORD_AUXTRACE: + case PERF_RECORD_AUXTRACE_INFO: + // These should be dealt with at tokenization time + PERFETTO_FATAL("Unexpected record type at parsing time: %" PRIu32, + record.header.type); + + default: + context_->storage->IncrementIndexedStats( + stats::perf_unknown_record_type, + static_cast<int>(record.header.type)); + return base::ErrStatus("Unknown PERF_RECORD with type %" PRIu32, + record.header.type); + } +} + +base::Status RecordParser::ParseSample(int64_t ts, Record record) { + Sample sample; + RETURN_IF_ERROR(sample.Parse(ts, record)); + + if (!sample.period.has_value() && record.attr != nullptr) { + sample.period = record.attr->sample_period(); + } + + return InternSample(std::move(sample)); +} + +base::Status RecordParser::InternSample(Sample sample) { + if (!sample.time.has_value()) { + // We do not really use this TS as this is using the perf clock, but we need + // it to be present so that we can compute the trace_ts done during + // tokenization. (Actually at tokenization time we do estimate a trace_ts if + // no perf ts is present, but for samples we want this to be as accurate as + // possible) + base::ErrStatus("Can not parse samples with no PERF_SAMPLE_TIME field"); + } + + if (!sample.pid_tid.has_value()) { + base::ErrStatus("Can not parse samples with no PERF_SAMPLE_TID field"); + } + + if (!sample.cpu.has_value()) { + base::ErrStatus("Can not parse samples with no PERF_SAMPLE_CPU field"); + } + + UniqueTid utid = context_->process_tracker->UpdateThread(sample.pid_tid->tid, + sample.pid_tid->pid); + const auto upid = *context_->storage->thread_table() + .FindById(tables::ThreadTable::Id(utid)) + ->upid(); + + if (sample.callchain.empty() && sample.ip.has_value()) { + sample.callchain.push_back(Sample::Frame{sample.cpu_mode, *sample.ip}); + } + std::optional<CallsiteId> callsite_id = + InternCallchain(upid, sample.callchain); + + context_->storage->mutable_perf_sample_table()->Insert( + {sample.trace_ts, utid, *sample.cpu, + context_->storage->InternString( + ProfilePacketUtils::StringifyCpuMode(sample.cpu_mode)), + callsite_id, std::nullopt, sample.perf_session->perf_session_id()}); + + return UpdateCounters(sample); +} + +std::optional<CallsiteId> RecordParser::InternCallchain( + UniquePid upid, + const std::vector<Sample::Frame>& callchain) { + if (callchain.empty()) { + return std::nullopt; + } + + auto& stack_profile_tracker = *context_->stack_profile_tracker; + auto& mapping_tracker = *context_->mapping_tracker; + + std::optional<CallsiteId> parent; + uint32_t depth = 0; + for (auto it = callchain.rbegin(); it != callchain.rend(); ++it) { + VirtualMemoryMapping* mapping; + if (IsInKernel(it->cpu_mode)) { + mapping = mapping_tracker.FindKernelMappingForAddress(it->ip); + } else { + mapping = mapping_tracker.FindUserMappingForAddress(upid, it->ip); + } + + if (!mapping) { + context_->storage->IncrementStats(stats::perf_dummy_mapping_used); + // Simpleperf will not create mappings for anonymous executable mappings + // which are used by JITted code (e.g. V8 JavaScript). + mapping = mapping_tracker.GetDummyMapping(); + } + + const FrameId frame_id = + mapping->InternFrame(mapping->ToRelativePc(it->ip), ""); + + parent = stack_profile_tracker.InternCallsite(parent, frame_id, depth); + depth++; + } + return parent; +} + +base::Status RecordParser::ParseComm(Record record) { + Reader reader(record.payload.copy()); + uint32_t pid; + uint32_t tid; + std::string comm; + if (!reader.Read(pid) || !reader.Read(tid) || !reader.ReadCString(comm)) { + return base::ErrStatus("Failed to parse PERF_RECORD_COMM"); + } + + context_->process_tracker->UpdateThread(tid, pid); + context_->process_tracker->UpdateThreadName( + tid, context_->storage->InternString(base::StringView(comm)), + ThreadNamePriority::kFtrace); + + return base::OkStatus(); +} + +base::Status RecordParser::ParseMmap(Record record) { + MmapRecord mmap; + RETURN_IF_ERROR(mmap.Parse(record)); + std::optional<BuildId> build_id = + record.session->LookupBuildId(mmap.pid, mmap.filename); + if (IsInKernel(record.GetCpuMode())) { + context_->mapping_tracker->CreateKernelMemoryMapping( + BuildCreateMappingParams(mmap, std::move(mmap.filename), + std::move(build_id))); + return base::OkStatus(); + } + + context_->mapping_tracker->CreateUserMemoryMapping( + GetUpid(mmap), BuildCreateMappingParams(mmap, std::move(mmap.filename), + std::move(build_id))); + + return base::OkStatus(); +} + +util::Status RecordParser::ParseMmap2(Record record) { + Mmap2Record mmap2; + RETURN_IF_ERROR(mmap2.Parse(record)); + std::optional<BuildId> build_id = mmap2.GetBuildId(); + if (!build_id.has_value()) { + build_id = record.session->LookupBuildId(mmap2.pid, mmap2.filename); + } + if (IsInKernel(record.GetCpuMode())) { + context_->mapping_tracker->CreateKernelMemoryMapping( + BuildCreateMappingParams(mmap2, std::move(mmap2.filename), + std::move(build_id))); + return base::OkStatus(); + } + + context_->mapping_tracker->CreateUserMemoryMapping( + GetUpid(mmap2), BuildCreateMappingParams(mmap2, std::move(mmap2.filename), + std::move(build_id))); + + return base::OkStatus(); +} + +UniquePid RecordParser::GetUpid(const CommonMmapRecordFields& fields) const { + UniqueTid utid = + context_->process_tracker->UpdateThread(fields.tid, fields.pid); + auto upid = context_->storage->thread_table() + .FindById(tables::ThreadTable::Id(utid)) + ->upid(); + PERFETTO_CHECK(upid.has_value()); + return *upid; +} + +base::Status RecordParser::UpdateCounters(const Sample& sample) { + if (!sample.read_groups.empty()) { + return UpdateCountersInReadGroups(sample); + } + + if (!sample.period.has_value() && !sample.attr->sample_period().has_value()) { + return base::ErrStatus("No period for sample"); + } + + uint64_t period = sample.period.has_value() ? *sample.period + : *sample.attr->sample_period(); + sample.attr->GetOrCreateCounter(*sample.cpu) + .AddDelta(sample.trace_ts, static_cast<double>(period)); + return base::OkStatus(); +} + +base::Status RecordParser::UpdateCountersInReadGroups(const Sample& sample) { + if (!sample.cpu.has_value()) { + return base::ErrStatus("No cpu for sample"); + } + + for (const auto& entry : sample.read_groups) { + RefPtr<const PerfEventAttr> attr = + sample.perf_session->FindAttrForEventId(*entry.event_id); + if (PERFETTO_UNLIKELY(!attr)) { + return base::ErrStatus("No perf_event_attr for id %" PRIu64, + *entry.event_id); + } + attr->GetOrCreateCounter(*sample.cpu) + .AddCount(sample.trace_ts, static_cast<double>(entry.value)); + } + return base::OkStatus(); +} + +} // namespace perf_importer +} // namespace trace_processor +} // namespace perfetto diff --git a/src/trace_processor/importers/perf/record_parser.h b/src/trace_processor/importers/perf/record_parser.h new file mode 100644 index 000000000..0a71b49a0 --- /dev/null +++ b/src/trace_processor/importers/perf/record_parser.h @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_RECORD_PARSER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_RECORD_PARSER_H_ + +#include <stdint.h> +#include <cstdint> +#include <vector> + +#include "perfetto/base/status.h" +#include "src/trace_processor/importers/common/trace_parser.h" +#include "src/trace_processor/importers/perf/mmap_record.h" +#include "src/trace_processor/importers/perf/record.h" +#include "src/trace_processor/importers/perf/sample.h" +#include "src/trace_processor/storage/trace_storage.h" + +namespace perfetto { +namespace trace_processor { + +class TraceProcessorContext; + +namespace perf_importer { + +class PerfDataTracker; +class Reader; + +// Parses samples from perf.data files. +class RecordParser : public PerfRecordParser { + public: + explicit RecordParser(TraceProcessorContext*); + ~RecordParser() override; + + void ParsePerfRecord(int64_t timestamp, Record record) override; + + private: + base::Status ParseRecord(int64_t timestamp, Record record); + base::Status ParseSample(int64_t ts, Record record); + base::Status ParseComm(Record record); + base::Status ParseMmap(Record record); + base::Status ParseMmap2(Record record); + + base::Status InternSample(Sample sample); + + base::Status UpdateCounters(const Sample& sample); + base::Status UpdateCountersInReadGroups(const Sample& sample); + + std::optional<CallsiteId> InternCallchain( + UniquePid upid, + const std::vector<Sample::Frame>& callchain); + + UniquePid GetUpid(const CommonMmapRecordFields& fields) const; + + TraceProcessorContext* context_ = nullptr; +}; + +} // namespace perf_importer +} // namespace trace_processor +} // namespace perfetto + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_RECORD_PARSER_H_ diff --git a/src/trace_processor/importers/perf/sample.cc b/src/trace_processor/importers/perf/sample.cc new file mode 100644 index 000000000..63ded53c9 --- /dev/null +++ b/src/trace_processor/importers/perf/sample.cc @@ -0,0 +1,252 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/perf/sample.h" + +#include <cstdint> + +#include "perfetto/base/logging.h" +#include "perfetto/base/status.h" +#include "perfetto/public/compiler.h" +#include "src/trace_processor/importers/perf/reader.h" +#include "src/trace_processor/importers/perf/record.h" + +namespace perfetto::trace_processor::perf_importer { +namespace { + +bool ParseSampleReadGroup(Reader& reader, + uint64_t read_format, + uint64_t num_records, + std::vector<Sample::ReadGroup>& out) { + out.resize(num_records); + for (auto& read : out) { + if (PERFETTO_UNLIKELY(!reader.Read(read.value))) { + return false; + } + + if (read_format & PERF_FORMAT_ID) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(read.event_id))) { + return false; + } + } + + if (read_format & PERF_FORMAT_LOST) { + uint64_t lost; + if (PERFETTO_UNLIKELY(!reader.Read(lost))) { + return false; + } + } + } + + return true; +} + +bool ParseSampleRead(Reader& reader, + uint64_t read_format, + std::vector<Sample::ReadGroup>& out) { + uint64_t value_or_nr; + + if (PERFETTO_UNLIKELY(!reader.Read(value_or_nr))) { + return false; + } + + if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { + uint64_t total_time_enabled; + if (PERFETTO_UNLIKELY(!reader.Read(total_time_enabled))) { + return false; + } + } + + if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { + uint64_t total_time_running; + if (PERFETTO_UNLIKELY(!reader.Read(total_time_running))) { + return false; + } + } + + if (read_format & PERF_FORMAT_GROUP) { + return ParseSampleReadGroup(reader, read_format, value_or_nr, out); + } + + std::optional<uint64_t> event_id; + if (read_format & PERF_FORMAT_ID) { + event_id.emplace(0); + if (PERFETTO_UNLIKELY(!reader.ReadOptional(event_id))) { + return false; + } + } + + if (read_format & PERF_FORMAT_LOST) { + uint64_t lost; + if (PERFETTO_UNLIKELY(!reader.Read(lost))) { + return false; + } + } + + out.push_back({event_id, value_or_nr}); + + return true; +} + +protos::pbzero::Profiling::CpuMode PerfCallchainContextToCpuMode(uint64_t ip) { + switch (ip) { + case PERF_CONTEXT_HV: + return protos::pbzero::Profiling::MODE_HYPERVISOR; + case PERF_CONTEXT_KERNEL: + return protos::pbzero::Profiling::MODE_KERNEL; + case PERF_CONTEXT_USER: + return protos::pbzero::Profiling::MODE_USER; + case PERF_CONTEXT_GUEST_KERNEL: + return protos::pbzero::Profiling::MODE_GUEST_KERNEL; + case PERF_CONTEXT_GUEST_USER: + return protos::pbzero::Profiling::MODE_GUEST_USER; + case PERF_CONTEXT_GUEST: + default: + return protos::pbzero::Profiling::MODE_UNKNOWN; + } + PERFETTO_FATAL("For GCC"); +} + +bool IsPerfContextMark(uint64_t ip) { + return ip >= PERF_CONTEXT_MAX; +} + +bool ParseSampleCallchain(Reader& reader, + protos::pbzero::Profiling::CpuMode cpu_mode, + std::vector<Sample::Frame>& out) { + uint64_t nr; + if (PERFETTO_UNLIKELY(!reader.Read(nr))) { + return false; + } + + std::vector<Sample::Frame> frames; + frames.reserve(nr); + for (; nr != 0; --nr) { + uint64_t ip; + if (PERFETTO_UNLIKELY(!reader.Read(ip))) { + return false; + } + if (PERFETTO_UNLIKELY(IsPerfContextMark(ip))) { + cpu_mode = PerfCallchainContextToCpuMode(ip); + continue; + } + frames.push_back({cpu_mode, ip}); + } + + out = std::move(frames); + return true; +} +} // namespace + +base::Status Sample::Parse(int64_t in_trace_ts, const Record& record) { + PERFETTO_CHECK(record.attr); + const uint64_t sample_type = record.attr->sample_type(); + + trace_ts = in_trace_ts; + cpu_mode = record.GetCpuMode(); + perf_session = record.session; + attr = record.attr; + + Reader reader(record.payload.copy()); + + std::optional<uint64_t> identifier; + if (sample_type & PERF_SAMPLE_IDENTIFIER) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(identifier))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_IDENTIFIER"); + } + } + + if (sample_type & PERF_SAMPLE_IP) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(ip))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_IP"); + } + } + + if (sample_type & PERF_SAMPLE_TID) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(pid_tid))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_TID"); + } + } + + if (sample_type & PERF_SAMPLE_TIME) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(time))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_TIME"); + } + } + + if (sample_type & PERF_SAMPLE_ADDR) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(addr))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_ADDR"); + } + } + + if (sample_type & PERF_SAMPLE_ID) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(id))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_ID"); + } + } + + if (identifier.has_value()) { + if (!id.has_value()) { + id = identifier; + } else if (PERFETTO_UNLIKELY(*identifier != *id)) { + return base::ErrStatus("ID and IDENTIFIER mismatch"); + } + } + + if (sample_type & PERF_SAMPLE_STREAM_ID) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(stream_id))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_STREAM_ID"); + } + } + + if (sample_type & PERF_SAMPLE_CPU) { + struct { + int32_t cpu; + int32_t unused; + } tmp; + if (PERFETTO_UNLIKELY(!reader.Read(tmp))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_CPU"); + } + cpu = tmp.cpu; + } + + if (sample_type & PERF_SAMPLE_PERIOD) { + if (PERFETTO_UNLIKELY(!reader.ReadOptional(period))) { + return base ::ErrStatus("Not enough data to read PERF_SAMPLE_PERIOD"); + } + } + + if (sample_type & PERF_SAMPLE_READ) { + if (PERFETTO_UNLIKELY( + !ParseSampleRead(reader, attr->read_format(), read_groups))) { + return base::ErrStatus("Failed to read PERF_SAMPLE_READ field"); + } + if (read_groups.empty()) { + return base::ErrStatus("No data in PERF_SAMPLE_READ field"); + } + } + + if (sample_type & PERF_SAMPLE_CALLCHAIN) { + if (PERFETTO_UNLIKELY(!ParseSampleCallchain(reader, cpu_mode, callchain))) { + return base::ErrStatus("Failed to read PERF_SAMPLE_CALLCHAIN field"); + } + } + + return base::OkStatus(); +} + +} // namespace perfetto::trace_processor::perf_importer diff --git a/src/trace_processor/importers/perf/sample.h b/src/trace_processor/importers/perf/sample.h new file mode 100644 index 000000000..532b6131f --- /dev/null +++ b/src/trace_processor/importers/perf/sample.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PERF_SAMPLE_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PERF_SAMPLE_H_ + +#include <cstdint> +#include <optional> +#include <vector> + +#include "perfetto/base/status.h" +#include "perfetto/trace_processor/ref_counted.h" +#include "protos/perfetto/trace/profiling/profile_packet.pbzero.h" +#include "src/trace_processor/importers/perf/perf_event_attr.h" +#include "src/trace_processor/importers/perf/perf_session.h" + +namespace perfetto::trace_processor::perf_importer { + +struct Record; + +struct Sample { + struct Frame { + protos::pbzero::Profiling::CpuMode cpu_mode; + uint64_t ip; + }; + + struct PidTid { + uint32_t pid; + uint32_t tid; + }; + + struct ReadGroup { + std::optional<uint64_t> event_id; + uint64_t value; + }; + + int64_t trace_ts; + protos::pbzero::Profiling::CpuMode cpu_mode; + RefPtr<PerfSession> perf_session; + RefPtr<const PerfEventAttr> attr; + + std::optional<uint64_t> ip; + std::optional<PidTid> pid_tid; + std::optional<uint64_t> time; + std::optional<uint64_t> addr; + std::optional<uint64_t> id; + std::optional<uint64_t> stream_id; + std::optional<uint32_t> cpu; + std::optional<uint64_t> period; + std::vector<ReadGroup> read_groups; + std::vector<Frame> callchain; + + base::Status Parse(int64_t trace_ts, const Record& record); +}; + +} // namespace perfetto::trace_processor::perf_importer + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PERF_SAMPLE_H_ diff --git a/src/trace_processor/importers/proto/BUILD.gn b/src/trace_processor/importers/proto/BUILD.gn index dde995cd9..effda8244 100644 --- a/src/trace_processor/importers/proto/BUILD.gn +++ b/src/trace_processor/importers/proto/BUILD.gn @@ -18,6 +18,8 @@ source_set("minimal") { sources = [ "active_chrome_processes_tracker.cc", "active_chrome_processes_tracker.h", + "args_parser.cc", + "args_parser.h", "chrome_string_lookup.cc", "chrome_string_lookup.h", "chrome_system_probes_module.cc", @@ -113,6 +115,8 @@ source_set("full") { "additional_modules.h", "android_camera_event_module.cc", "android_camera_event_module.h", + "android_input_event_module.cc", + "android_input_event_module.h", "android_probes_module.cc", "android_probes_module.h", "android_probes_parser.cc", diff --git a/src/trace_processor/importers/proto/additional_modules.cc b/src/trace_processor/importers/proto/additional_modules.cc index 91645c393..cf2ee84ed 100644 --- a/src/trace_processor/importers/proto/additional_modules.cc +++ b/src/trace_processor/importers/proto/additional_modules.cc @@ -18,6 +18,7 @@ #include "src/trace_processor/importers/etw/etw_module_impl.h" #include "src/trace_processor/importers/ftrace/ftrace_module_impl.h" #include "src/trace_processor/importers/proto/android_camera_event_module.h" +#include "src/trace_processor/importers/proto/android_input_event_module.h" #include "src/trace_processor/importers/proto/android_probes_module.h" #include "src/trace_processor/importers/proto/graphics_event_module.h" #include "src/trace_processor/importers/proto/heap_graph_module.h" @@ -45,6 +46,7 @@ void RegisterAdditionalModules(TraceProcessorContext* context) { context->modules.emplace_back(new MetadataModule(context)); context->modules.emplace_back(new V8Module(context)); context->modules.emplace_back(new WinscopeModule(context)); + context->modules.emplace_back(new AndroidInputEventModule(context)); // Ftrace/Etw modules are special, because it has one extra method for parsing // ftrace/etw packets. So we need to store a pointer to it separately. diff --git a/src/trace_processor/importers/proto/android_input_event_module.cc b/src/trace_processor/importers/proto/android_input_event_module.cc new file mode 100644 index 000000000..26671a77f --- /dev/null +++ b/src/trace_processor/importers/proto/android_input_event_module.cc @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/proto/android_input_event_module.h" + +#include "protos/perfetto/trace/android/android_input_event.pbzero.h" +#include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/proto/args_parser.h" +#include "src/trace_processor/importers/proto/trace.descriptor.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/android_tables_py.h" +#include "src/trace_processor/types/trace_processor_context.h" + +namespace perfetto::trace_processor { + +using perfetto::protos::pbzero::AndroidInputEvent; +using perfetto::protos::pbzero::AndroidKeyEvent; +using perfetto::protos::pbzero::AndroidMotionEvent; +using perfetto::protos::pbzero::AndroidWindowInputDispatchEvent; +using perfetto::protos::pbzero::TracePacket; + +AndroidInputEventModule::AndroidInputEventModule(TraceProcessorContext* context) + : context_(*context), args_parser_(pool_) { + pool_.AddFromFileDescriptorSet(kTraceDescriptor.data(), + kTraceDescriptor.size()); + RegisterForField(TracePacket::kAndroidInputEventFieldNumber, context); +} + +void AndroidInputEventModule::ParseTracePacketData( + const TracePacket::Decoder& decoder, + int64_t packet_ts, + const TracePacketData&, + uint32_t field_id) { + if (field_id != TracePacket::kAndroidInputEventFieldNumber) + return; + + auto input_event = AndroidInputEvent::Decoder(decoder.android_input_event()); + + constexpr static auto supported_fields = std::array{ + AndroidInputEvent::kDispatcherMotionEventFieldNumber, + AndroidInputEvent::kDispatcherMotionEventRedactedFieldNumber, + AndroidInputEvent::kDispatcherKeyEventFieldNumber, + AndroidInputEvent::kDispatcherKeyEventRedactedFieldNumber, + AndroidInputEvent::kDispatcherWindowDispatchEventFieldNumber, + AndroidInputEvent::kDispatcherWindowDispatchEventRedactedFieldNumber}; + + for (auto sub_field_id : supported_fields) { + auto sub_field = input_event.Get(static_cast<uint32_t>(sub_field_id)); + if (!sub_field.valid()) + continue; + + switch (sub_field_id) { + case AndroidInputEvent::kDispatcherMotionEventFieldNumber: + case AndroidInputEvent::kDispatcherMotionEventRedactedFieldNumber: + ParseMotionEvent(packet_ts, sub_field.as_bytes()); + return; + case AndroidInputEvent::kDispatcherKeyEventFieldNumber: + case AndroidInputEvent::kDispatcherKeyEventRedactedFieldNumber: + ParseKeyEvent(packet_ts, sub_field.as_bytes()); + return; + case AndroidInputEvent::kDispatcherWindowDispatchEventFieldNumber: + case AndroidInputEvent::kDispatcherWindowDispatchEventRedactedFieldNumber: + ParseWindowDispatchEvent(packet_ts, sub_field.as_bytes()); + return; + } + } +} + +void AndroidInputEventModule::ParseMotionEvent( + int64_t packet_ts, + const protozero::ConstBytes& bytes) { + AndroidMotionEvent::Decoder event_proto(bytes); + tables::AndroidMotionEventsTable::Row event_row; + event_row.event_id = event_proto.event_id(); + event_row.ts = event_proto.event_time_nanos(); + + auto event_row_id = context_.storage->mutable_android_motion_events_table() + ->Insert(event_row) + .id; + auto inserter = context_.args_tracker->AddArgsTo(event_row_id); + ArgsParser writer{packet_ts, inserter, *context_.storage}; + + base::Status status = + args_parser_.ParseMessage(bytes, ".perfetto.protos.AndroidMotionEvent", + nullptr /*parse all fields*/, writer); + if (!status.ok()) + context_.storage->IncrementStats(stats::android_input_event_parse_errors); +} + +void AndroidInputEventModule::ParseKeyEvent( + int64_t packet_ts, + const protozero::ConstBytes& bytes) { + AndroidKeyEvent::Decoder event_proto(bytes); + tables::AndroidKeyEventsTable::Row event_row; + event_row.event_id = event_proto.event_id(); + event_row.ts = event_proto.event_time_nanos(); + + auto event_row_id = context_.storage->mutable_android_key_events_table() + ->Insert(event_row) + .id; + auto inserter = context_.args_tracker->AddArgsTo(event_row_id); + ArgsParser writer{packet_ts, inserter, *context_.storage}; + + base::Status status = + args_parser_.ParseMessage(bytes, ".perfetto.protos.AndroidKeyEvent", + nullptr /*parse all fields*/, writer); + if (!status.ok()) + context_.storage->IncrementStats(stats::android_input_event_parse_errors); +} + +void AndroidInputEventModule::ParseWindowDispatchEvent( + int64_t packet_ts, + const protozero::ConstBytes& bytes) { + AndroidWindowInputDispatchEvent::Decoder event_proto(bytes); + tables::AndroidInputEventDispatchTable::Row event_row; + event_row.event_id = event_proto.event_id(); + event_row.vsync_id = event_proto.vsync_id(); + event_row.window_id = event_proto.window_id(); + + auto event_row_id = + context_.storage->mutable_android_input_event_dispatch_table() + ->Insert(event_row) + .id; + auto inserter = context_.args_tracker->AddArgsTo(event_row_id); + ArgsParser writer{packet_ts, inserter, *context_.storage}; + + base::Status status = args_parser_.ParseMessage( + bytes, ".perfetto.protos.AndroidWindowInputDispatchEvent", + nullptr /*parse all fields*/, writer); + if (!status.ok()) + context_.storage->IncrementStats(stats::android_input_event_parse_errors); +} + +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/proto/android_input_event_module.h b/src/trace_processor/importers/proto/android_input_event_module.h new file mode 100644 index 000000000..c31090909 --- /dev/null +++ b/src/trace_processor/importers/proto/android_input_event_module.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_ANDROID_INPUT_EVENT_MODULE_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_ANDROID_INPUT_EVENT_MODULE_H_ + +#include <cstdint> +#include "perfetto/base/build_config.h" +#include "protos/perfetto/trace/trace_packet.pbzero.h" +#include "src/trace_processor/importers/common/parser_types.h" +#include "src/trace_processor/importers/proto/proto_importer_module.h" +#include "src/trace_processor/util/descriptors.h" +#include "src/trace_processor/util/proto_to_args_parser.h" + +namespace perfetto::trace_processor { + +class AndroidInputEventModule : public ProtoImporterModule { + public: + explicit AndroidInputEventModule(TraceProcessorContext* context); + + void ParseTracePacketData(const protos::pbzero::TracePacket::Decoder&, + int64_t packet_ts, + const TracePacketData&, + uint32_t field_id) override; + + private: + TraceProcessorContext& context_; + DescriptorPool pool_; + util::ProtoToArgsParser args_parser_; + + void ParseMotionEvent(int64_t packet_ts, const protozero::ConstBytes& bytes); + void ParseKeyEvent(int64_t packet_ts, const protozero::ConstBytes& bytes); + void ParseWindowDispatchEvent(int64_t packet_ts, const protozero::ConstBytes& bytes); +}; + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_ANDROID_INPUT_EVENT_MODULE_H_ diff --git a/src/trace_processor/importers/proto/args_parser.cc b/src/trace_processor/importers/proto/args_parser.cc new file mode 100644 index 000000000..2aeb785c4 --- /dev/null +++ b/src/trace_processor/importers/proto/args_parser.cc @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/proto/args_parser.h" + +#include "perfetto/ext/base/base64.h" +#include "src/trace_processor/importers/json/json_utils.h" + +namespace perfetto::trace_processor { + +using BoundInserter = ArgsTracker::BoundInserter; + +ArgsParser::ArgsParser(int64_t packet_timestamp, + BoundInserter& inserter, + TraceStorage& storage, + PacketSequenceStateGeneration* sequence_state, + bool support_json) + : support_json_(support_json), + packet_timestamp_(packet_timestamp), + sequence_state_(sequence_state), + inserter_(inserter), + storage_(storage) {} + +ArgsParser::~ArgsParser() = default; + +void ArgsParser::AddInteger(const Key& key, int64_t value) { + inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::Integer(value)); +} + +void ArgsParser::AddUnsignedInteger(const Key& key, uint64_t value) { + inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::UnsignedInteger(value)); +} + +void ArgsParser::AddString(const Key& key, const protozero::ConstChars& value) { + inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::String(storage_.InternString(value))); +} + +void ArgsParser::AddString(const Key& key, const std::string& value) { + inserter_.AddArg( + storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::String(storage_.InternString(base::StringView(value)))); +} + +void ArgsParser::AddDouble(const Key& key, double value) { + inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::Real(value)); +} + +void ArgsParser::AddPointer(const Key& key, const void* value) { + inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::Pointer(reinterpret_cast<uintptr_t>(value))); +} + +void ArgsParser::AddBoolean(const Key& key, bool value) { + inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::Boolean(value)); +} + +void ArgsParser::AddBytes(const Key& key, const protozero::ConstBytes& value) { + std::string b64_data = base::Base64Encode(value.data, value.size); + AddString(key, b64_data); +} + +bool ArgsParser::AddJson(const Key& key, const protozero::ConstChars& value) { + if (!support_json_) + PERFETTO_FATAL("Unexpected JSON value when parsing data"); + + auto json_value = json::ParseJsonString(value); + if (!json_value) + return false; + return json::AddJsonValueToArgs(*json_value, base::StringView(key.flat_key), + base::StringView(key.key), &storage_, + &inserter_); +} + +void ArgsParser::AddNull(const Key& key) { + inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), + storage_.InternString(base::StringView(key.key)), + Variadic::Null()); +} + +size_t ArgsParser::GetArrayEntryIndex(const std::string& array_key) { + return inserter_.GetNextArrayEntryIndex( + storage_.InternString(base::StringView(array_key))); +} + +size_t ArgsParser::IncrementArrayEntryIndex(const std::string& array_key) { + return inserter_.IncrementArrayEntryIndex( + storage_.InternString(base::StringView(array_key))); +} + +int64_t ArgsParser::packet_timestamp() { + return packet_timestamp_; +} + +PacketSequenceStateGeneration* ArgsParser::seq_state() { + return sequence_state_; +} + +InternedMessageView* ArgsParser::GetInternedMessageView(uint32_t field_id, + uint64_t iid) { + if (!sequence_state_) + return nullptr; + return sequence_state_->GetInternedMessageView(field_id, iid); +} + +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/proto/winscope/winscope_args_parser.h b/src/trace_processor/importers/proto/args_parser.h index a0dfa4632..7cd157848 100644 --- a/src/trace_processor/importers/proto/winscope/winscope_args_parser.h +++ b/src/trace_processor/importers/proto/args_parser.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 The Android Open Source Project + * Copyright (C) 2024 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,22 +14,28 @@ * limitations under the License. */ -#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_WINSCOPE_WINSCOPE_ARGS_PARSER_H_ -#define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_WINSCOPE_WINSCOPE_ARGS_PARSER_H_ +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_ARGS_PARSER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_ARGS_PARSER_H_ #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/proto/packet_sequence_state_generation.h" #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/util/proto_to_args_parser.h" -namespace perfetto { -namespace trace_processor { +namespace perfetto::trace_processor { -class WinscopeArgsParser : public util::ProtoToArgsParser::Delegate { +// A ProtoToArgsParser::Delegate that writes the parsed proto data into +// TraceStorage after interning key strings. +class ArgsParser : public util::ProtoToArgsParser::Delegate { public: using Key = util::ProtoToArgsParser::Key; - WinscopeArgsParser(ArgsTracker::BoundInserter& inserter, - TraceStorage& storage); + ArgsParser(int64_t packet_timestamp, + ArgsTracker::BoundInserter& inserter, + TraceStorage& storage, + PacketSequenceStateGeneration* sequence_state = nullptr, + bool support_json = false); + ~ArgsParser() override; void AddInteger(const Key&, int64_t) override; void AddUnsignedInteger(const Key&, uint64_t) override; void AddString(const Key&, const protozero::ConstChars&) override; @@ -37,10 +43,12 @@ class WinscopeArgsParser : public util::ProtoToArgsParser::Delegate { void AddDouble(const Key&, double) override; void AddPointer(const Key&, const void*) override; void AddBoolean(const Key&, bool) override; + void AddBytes(const Key&, const protozero::ConstBytes&) override; bool AddJson(const Key&, const protozero::ConstChars&) override; void AddNull(const Key&) override; size_t GetArrayEntryIndex(const std::string& array_key) override; size_t IncrementArrayEntryIndex(const std::string& array_key) override; + int64_t packet_timestamp() override; PacketSequenceStateGeneration* seq_state() override; protected: @@ -48,11 +56,13 @@ class WinscopeArgsParser : public util::ProtoToArgsParser::Delegate { uint64_t iid) override; private: + const bool support_json_; + const int64_t packet_timestamp_; + PacketSequenceStateGeneration* const sequence_state_; ArgsTracker::BoundInserter& inserter_; TraceStorage& storage_; }; -} // namespace trace_processor -} // namespace perfetto +} // namespace perfetto::trace_processor -#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_WINSCOPE_WINSCOPE_ARGS_PARSER_H_ +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_ARGS_PARSER_H_ diff --git a/src/trace_processor/importers/proto/chrome_system_probes_parser.h b/src/trace_processor/importers/proto/chrome_system_probes_parser.h index ca688146b..9c9397b7f 100644 --- a/src/trace_processor/importers/proto/chrome_system_probes_parser.h +++ b/src/trace_processor/importers/proto/chrome_system_probes_parser.h @@ -43,7 +43,7 @@ class ChromeSystemProbesParser { // Maps a proto field number for memcounters in ProcessStats::Process to // their StringId. Keep kProcStatsProcessSize equal to 1 + max proto field // id of ProcessStats::Process. Also update SystemProbesParser. - static constexpr size_t kProcStatsProcessSize = 23; + static constexpr size_t kProcStatsProcessSize = 24; std::array<StringId, kProcStatsProcessSize> proc_stats_process_names_{}; }; diff --git a/src/trace_processor/importers/proto/gpu_event_parser.cc b/src/trace_processor/importers/proto/gpu_event_parser.cc index 9022cbb02..5664fba8c 100644 --- a/src/trace_processor/importers/proto/gpu_event_parser.cc +++ b/src/trace_processor/importers/proto/gpu_event_parser.cc @@ -330,6 +330,7 @@ void GpuEventParser::ParseGpuRenderStageEvent( ConstBytes blob) { protos::pbzero::GpuRenderStageEvent::Decoder event(blob.data, blob.size); + int32_t pid = 0; if (event.has_specifications()) { protos::pbzero::GpuRenderStageEvent_Specifications::Decoder spec( event.specifications().data, event.specifications().size); @@ -349,6 +350,23 @@ void GpuEventParser::ParseGpuRenderStageEvent( context_->storage->InternString(stage.description()))); } } + if (spec.has_context_spec()) { + protos::pbzero::GpuRenderStageEvent_Specifications_ContextSpec::Decoder + context_spec(spec.context_spec()); + if (context_spec.has_pid()) { + pid = context_spec.pid(); + } + } + } + + if (event.has_context()) { + uint64_t context_id = event.context(); + auto* decoder = sequence_state->LookupInternedMessage< + protos::pbzero::InternedData::kGraphicsContextsFieldNumber, + protos::pbzero::InternedGraphicsContext>(context_id); + if (decoder) { + pid = decoder->pid(); + } } auto args_callback = [this, &event, @@ -468,7 +486,8 @@ void GpuEventParser::ParseGpuRenderStageEvent( row.command_buffer_name = command_buffer_name_id; row.submission_id = event.submission_id(); row.hw_queue_id = static_cast<int64_t>(hw_queue_id); - + row.upid = context_->process_tracker->GetOrCreateProcess( + static_cast<uint32_t>(pid)); context_->slice_tracker->ScopedTyped( context_->storage->mutable_gpu_slice_table(), row, args_callback); } diff --git a/src/trace_processor/importers/proto/network_trace_module_unittest.cc b/src/trace_processor/importers/proto/network_trace_module_unittest.cc index f174b352a..017b7f752 100644 --- a/src/trace_processor/importers/proto/network_trace_module_unittest.cc +++ b/src/trace_processor/importers/proto/network_trace_module_unittest.cc @@ -20,6 +20,7 @@ #include "src/trace_processor/importers/common/args_translation_table.h" #include "src/trace_processor/importers/common/async_track_set_tracker.h" #include "src/trace_processor/importers/common/global_args_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/slice_tracker.h" #include "src/trace_processor/importers/common/slice_translation_table.h" #include "src/trace_processor/importers/common/track_tracker.h" @@ -45,6 +46,8 @@ class NetworkTraceModuleTest : public testing::Test { context_.args_tracker.reset(new ArgsTracker(&context_)); context_.global_args_tracker.reset(new GlobalArgsTracker(storage_)); context_.slice_translation_table.reset(new SliceTranslationTable(storage_)); + context_.process_track_translation_table.reset( + new ProcessTrackTranslationTable(storage_)); context_.args_translation_table.reset(new ArgsTranslationTable(storage_)); context_.async_track_set_tracker.reset(new AsyncTrackSetTracker(&context_)); context_.proto_trace_parser.reset(new ProtoTraceParserImpl(&context_)); diff --git a/src/trace_processor/importers/proto/perf_sample_tracker.cc b/src/trace_processor/importers/proto/perf_sample_tracker.cc index e47239e1c..52ff1152b 100644 --- a/src/trace_processor/importers/proto/perf_sample_tracker.cc +++ b/src/trace_processor/importers/proto/perf_sample_tracker.cc @@ -129,7 +129,7 @@ PerfSampleTracker::SamplingStreamInfo PerfSampleTracker::GetSamplingStreamInfo( seq_it = seq_state_.emplace(seq_id, CreatePerfSession()).first; } SequenceState* seq_state = &seq_it->second; - uint32_t session_id = seq_state->perf_session_id; + tables::PerfSessionTable::Id session_id = seq_state->perf_session_id; auto cpu_it = seq_state->per_cpu.find(cpu); if (cpu_it != seq_state->per_cpu.end()) @@ -162,15 +162,19 @@ PerfSampleTracker::SamplingStreamInfo PerfSampleTracker::GetSamplingStreamInfo( // tracing session). if (perf_defaults.has_value() && perf_defaults->process_shard_count() > 0) { context_->storage->SetIndexedStats( - stats::perf_process_shard_count, static_cast<int>(session_id), + stats::perf_process_shard_count, static_cast<int>(session_id.value), static_cast<int64_t>(perf_defaults->process_shard_count())); context_->storage->SetIndexedStats( - stats::perf_chosen_process_shard, static_cast<int>(session_id), + stats::perf_chosen_process_shard, static_cast<int>(session_id.value), static_cast<int64_t>(perf_defaults->chosen_process_shard())); } return {session_id, timebase_track_id}; } +tables::PerfSessionTable::Id PerfSampleTracker::CreatePerfSession() { + return context_->storage->mutable_perf_session_table()->Insert({}).id; +} + } // namespace trace_processor } // namespace perfetto diff --git a/src/trace_processor/importers/proto/perf_sample_tracker.h b/src/trace_processor/importers/proto/perf_sample_tracker.h index c1e62e25a..178683db3 100644 --- a/src/trace_processor/importers/proto/perf_sample_tracker.h +++ b/src/trace_processor/importers/proto/perf_sample_tracker.h @@ -23,6 +23,7 @@ #include <unordered_map> #include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/tables/profiler_tables_py.h" namespace perfetto { namespace protos { @@ -36,10 +37,11 @@ class TraceProcessorContext; class PerfSampleTracker { public: struct SamplingStreamInfo { - uint32_t perf_session_id = 0; + tables::PerfSessionTable::Id perf_session_id; TrackId timebase_track_id = kInvalidTrackId; - SamplingStreamInfo(uint32_t _perf_session_id, TrackId _timebase_track_id) + SamplingStreamInfo(tables::PerfSessionTable::Id _perf_session_id, + TrackId _timebase_track_id) : perf_session_id(_perf_session_id), timebase_track_id(_timebase_track_id) {} }; @@ -52,8 +54,6 @@ class PerfSampleTracker { uint32_t cpu, protos::pbzero::TracePacketDefaults_Decoder* nullable_defaults); - uint32_t CreatePerfSession() { return next_perf_session_id_++; } - private: struct CpuSequenceState { TrackId timebase_track_id = kInvalidTrackId; @@ -63,15 +63,16 @@ class PerfSampleTracker { }; struct SequenceState { - uint32_t perf_session_id = 0; + tables::PerfSessionTable::Id perf_session_id; std::unordered_map<uint32_t, CpuSequenceState> per_cpu; - SequenceState(uint32_t _perf_session_id) + explicit SequenceState(tables::PerfSessionTable::Id _perf_session_id) : perf_session_id(_perf_session_id) {} }; + tables::PerfSessionTable::Id CreatePerfSession(); + std::unordered_map<uint32_t, SequenceState> seq_state_; - uint32_t next_perf_session_id_ = 0; TraceProcessorContext* const context_; }; diff --git a/src/trace_processor/importers/proto/perf_sample_tracker_unittest.cc b/src/trace_processor/importers/proto/perf_sample_tracker_unittest.cc index 7515a04a5..df1fba0cc 100644 --- a/src/trace_processor/importers/proto/perf_sample_tracker_unittest.cc +++ b/src/trace_processor/importers/proto/perf_sample_tracker_unittest.cc @@ -215,10 +215,10 @@ TEST_F(PerfSampleTrackerTest, ProcessShardingStatsEntries) { std::optional<int64_t> shard_count = context.storage->GetIndexedStats( stats::perf_process_shard_count, - static_cast<int>(stream.perf_session_id)); + static_cast<int>(stream.perf_session_id.value)); std::optional<int64_t> chosen_shard = context.storage->GetIndexedStats( stats::perf_chosen_process_shard, - static_cast<int>(stream.perf_session_id)); + static_cast<int>(stream.perf_session_id.value)); ASSERT_TRUE(shard_count.has_value()); EXPECT_EQ(shard_count.value(), 8); @@ -227,10 +227,10 @@ TEST_F(PerfSampleTrackerTest, ProcessShardingStatsEntries) { std::optional<int64_t> shard_count2 = context.storage->GetIndexedStats( stats::perf_process_shard_count, - static_cast<int>(stream.perf_session_id)); + static_cast<int>(stream.perf_session_id.value)); std::optional<int64_t> chosen_shard2 = context.storage->GetIndexedStats( stats::perf_chosen_process_shard, - static_cast<int>(stream.perf_session_id)); + static_cast<int>(stream.perf_session_id.value)); ASSERT_TRUE(shard_count2.has_value()); EXPECT_EQ(shard_count2.value(), 8); diff --git a/src/trace_processor/importers/proto/profile_module.cc b/src/trace_processor/importers/proto/profile_module.cc index 55e3547bd..ab44d2a3e 100644 --- a/src/trace_processor/importers/proto/profile_module.cc +++ b/src/trace_processor/importers/proto/profile_module.cc @@ -238,7 +238,7 @@ void ProfileModule::ParsePerfSample( PerfSample::ProducerEvent::PROFILER_STOP_GUARDRAIL) { context_->storage->SetIndexedStats( stats::perf_guardrail_stop_ts, - static_cast<int>(sampling_stream.perf_session_id), ts); + static_cast<int>(sampling_stream.perf_session_id.value), ts); } return; } diff --git a/src/trace_processor/importers/proto/proto_trace_parser_impl.cc b/src/trace_processor/importers/proto/proto_trace_parser_impl.cc index 9d318a1c8..8a11d5d8a 100644 --- a/src/trace_processor/importers/proto/proto_trace_parser_impl.cc +++ b/src/trace_processor/importers/proto/proto_trace_parser_impl.cc @@ -29,8 +29,8 @@ #include "perfetto/ext/base/uuid.h" #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/event_tracker.h" -#include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/metadata_tracker.h" #include "src/trace_processor/importers/common/parser_types.h" #include "src/trace_processor/importers/common/process_tracker.h" @@ -44,7 +44,6 @@ #include "src/trace_processor/types/trace_processor_context.h" #include "src/trace_processor/types/variadic.h" -#include "protos/perfetto/common/trace_stats.pbzero.h" #include "protos/perfetto/config/trace_config.pbzero.h" #include "protos/perfetto/trace/chrome/chrome_trace_event.pbzero.h" #include "protos/perfetto/trace/perfetto/perfetto_metatrace.pbzero.h" @@ -85,9 +84,6 @@ void ProtoTraceParserImpl::ParseTracePacket(int64_t ts, TracePacketData data) { } } - if (packet.has_trace_stats()) - ParseTraceStats(packet.trace_stats()); - if (packet.has_chrome_events()) { ParseChromeEvents(ts, packet.chrome_events()); } @@ -160,116 +156,14 @@ void ProtoTraceParserImpl::ParseInlineSchedWaking(uint32_t cpu, context_->args_tracker->Flush(); } -void ProtoTraceParserImpl::ParseTraceStats(ConstBytes blob) { - protos::pbzero::TraceStats::Decoder evt(blob.data, blob.size); - auto* storage = context_->storage.get(); - storage->SetStats(stats::traced_producers_connected, - static_cast<int64_t>(evt.producers_connected())); - storage->SetStats(stats::traced_producers_seen, - static_cast<int64_t>(evt.producers_seen())); - storage->SetStats(stats::traced_data_sources_registered, - static_cast<int64_t>(evt.data_sources_registered())); - storage->SetStats(stats::traced_data_sources_seen, - static_cast<int64_t>(evt.data_sources_seen())); - storage->SetStats(stats::traced_tracing_sessions, - static_cast<int64_t>(evt.tracing_sessions())); - storage->SetStats(stats::traced_total_buffers, - static_cast<int64_t>(evt.total_buffers())); - storage->SetStats(stats::traced_chunks_discarded, - static_cast<int64_t>(evt.chunks_discarded())); - storage->SetStats(stats::traced_patches_discarded, - static_cast<int64_t>(evt.patches_discarded())); - storage->SetStats(stats::traced_flushes_requested, - static_cast<int64_t>(evt.flushes_requested())); - storage->SetStats(stats::traced_flushes_succeeded, - static_cast<int64_t>(evt.flushes_succeeded())); - storage->SetStats(stats::traced_flushes_failed, - static_cast<int64_t>(evt.flushes_failed())); - - if (evt.has_filter_stats()) { - protos::pbzero::TraceStats::FilterStats::Decoder fstat(evt.filter_stats()); - storage->SetStats(stats::filter_errors, - static_cast<int64_t>(fstat.errors())); - storage->SetStats(stats::filter_input_bytes, - static_cast<int64_t>(fstat.input_bytes())); - storage->SetStats(stats::filter_input_packets, - static_cast<int64_t>(fstat.input_packets())); - storage->SetStats(stats::filter_output_bytes, - static_cast<int64_t>(fstat.output_bytes())); - storage->SetStats(stats::filter_time_taken_ns, - static_cast<int64_t>(fstat.time_taken_ns())); - for (auto [i, it] = std::tuple{0, fstat.bytes_discarded_per_buffer()}; it; - ++it, ++i) { - storage->SetIndexedStats(stats::traced_buf_bytes_filtered_out, i, - static_cast<int64_t>(*it)); - } - } - - switch (evt.final_flush_outcome()) { - case protos::pbzero::TraceStats::FINAL_FLUSH_SUCCEEDED: - storage->IncrementStats(stats::traced_final_flush_succeeded, 1); - break; - case protos::pbzero::TraceStats::FINAL_FLUSH_FAILED: - storage->IncrementStats(stats::traced_final_flush_failed, 1); - break; - case protos::pbzero::TraceStats::FINAL_FLUSH_UNSPECIFIED: - break; - } - - int buf_num = 0; - for (auto it = evt.buffer_stats(); it; ++it, ++buf_num) { - protos::pbzero::TraceStats::BufferStats::Decoder buf(*it); - storage->SetIndexedStats(stats::traced_buf_buffer_size, buf_num, - static_cast<int64_t>(buf.buffer_size())); - storage->SetIndexedStats(stats::traced_buf_bytes_written, buf_num, - static_cast<int64_t>(buf.bytes_written())); - storage->SetIndexedStats(stats::traced_buf_bytes_overwritten, buf_num, - static_cast<int64_t>(buf.bytes_overwritten())); - storage->SetIndexedStats(stats::traced_buf_bytes_read, buf_num, - static_cast<int64_t>(buf.bytes_read())); - storage->SetIndexedStats(stats::traced_buf_padding_bytes_written, buf_num, - static_cast<int64_t>(buf.padding_bytes_written())); - storage->SetIndexedStats(stats::traced_buf_padding_bytes_cleared, buf_num, - static_cast<int64_t>(buf.padding_bytes_cleared())); - storage->SetIndexedStats(stats::traced_buf_chunks_written, buf_num, - static_cast<int64_t>(buf.chunks_written())); - storage->SetIndexedStats(stats::traced_buf_chunks_rewritten, buf_num, - static_cast<int64_t>(buf.chunks_rewritten())); - storage->SetIndexedStats(stats::traced_buf_chunks_overwritten, buf_num, - static_cast<int64_t>(buf.chunks_overwritten())); - storage->SetIndexedStats(stats::traced_buf_chunks_discarded, buf_num, - static_cast<int64_t>(buf.chunks_discarded())); - storage->SetIndexedStats(stats::traced_buf_chunks_read, buf_num, - static_cast<int64_t>(buf.chunks_read())); - storage->SetIndexedStats( - stats::traced_buf_chunks_committed_out_of_order, buf_num, - static_cast<int64_t>(buf.chunks_committed_out_of_order())); - storage->SetIndexedStats(stats::traced_buf_write_wrap_count, buf_num, - static_cast<int64_t>(buf.write_wrap_count())); - storage->SetIndexedStats(stats::traced_buf_patches_succeeded, buf_num, - static_cast<int64_t>(buf.patches_succeeded())); - storage->SetIndexedStats(stats::traced_buf_patches_failed, buf_num, - static_cast<int64_t>(buf.patches_failed())); - storage->SetIndexedStats(stats::traced_buf_readaheads_succeeded, buf_num, - static_cast<int64_t>(buf.readaheads_succeeded())); - storage->SetIndexedStats(stats::traced_buf_readaheads_failed, buf_num, - static_cast<int64_t>(buf.readaheads_failed())); - storage->SetIndexedStats(stats::traced_buf_abi_violations, buf_num, - static_cast<int64_t>(buf.abi_violations())); - storage->SetIndexedStats( - stats::traced_buf_trace_writer_packet_loss, buf_num, - static_cast<int64_t>(buf.trace_writer_packet_loss())); - } -} - void ProtoTraceParserImpl::ParseChromeEvents(int64_t ts, ConstBytes blob) { TraceStorage* storage = context_->storage.get(); protos::pbzero::ChromeEventBundle::Decoder bundle(blob.data, blob.size); ArgsTracker args(context_); if (bundle.has_metadata()) { + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(0); RawId id = storage->mutable_raw_table() - ->Insert({ts, raw_chrome_metadata_event_id_, 0, 0, 0, 0, - context_->machine_id()}) + ->Insert({ts, raw_chrome_metadata_event_id_, 0, 0, 0, ucpu}) .id; auto inserter = args.AddArgsTo(id); @@ -315,9 +209,10 @@ void ProtoTraceParserImpl::ParseChromeEvents(int64_t ts, ConstBytes blob) { } if (bundle.has_legacy_ftrace_output()) { + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(0); RawId id = storage->mutable_raw_table() ->Insert({ts, raw_chrome_legacy_system_trace_event_id_, 0, 0, - 0, 0, context_->machine_id()}) + 0, ucpu}) .id; std::string data; @@ -336,9 +231,10 @@ void ProtoTraceParserImpl::ParseChromeEvents(int64_t ts, ConstBytes blob) { protos::pbzero::ChromeLegacyJsonTrace::USER_TRACE) { continue; } + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(0); RawId id = storage->mutable_raw_table() ->Insert({ts, raw_chrome_legacy_user_trace_event_id_, 0, 0, - 0, 0, context_->machine_id()}) + 0, ucpu}) .id; Variadic value = Variadic::String(storage->InternString(legacy_trace.data())); diff --git a/src/trace_processor/importers/proto/proto_trace_parser_impl.h b/src/trace_processor/importers/proto/proto_trace_parser_impl.h index e9bce35b3..2c4dc077f 100644 --- a/src/trace_processor/importers/proto/proto_trace_parser_impl.h +++ b/src/trace_processor/importers/proto/proto_trace_parser_impl.h @@ -65,7 +65,6 @@ class ProtoTraceParserImpl : public ProtoTraceParser { int64_t /*ts*/, InlineSchedWaking data) override; - void ParseTraceStats(ConstBytes); void ParseChromeEvents(int64_t ts, ConstBytes); void ParseMetatraceEvent(int64_t ts, ConstBytes); diff --git a/src/trace_processor/importers/proto/proto_trace_parser_impl_unittest.cc b/src/trace_processor/importers/proto/proto_trace_parser_impl_unittest.cc index 95b16d130..e959f59e4 100644 --- a/src/trace_processor/importers/proto/proto_trace_parser_impl_unittest.cc +++ b/src/trace_processor/importers/proto/proto_trace_parser_impl_unittest.cc @@ -23,10 +23,13 @@ #include "src/trace_processor/importers/common/args_tracker.h" #include "src/trace_processor/importers/common/args_translation_table.h" #include "src/trace_processor/importers/common/clock_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/event_tracker.h" #include "src/trace_processor/importers/common/flow_tracker.h" +#include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/mapping_tracker.h" #include "src/trace_processor/importers/common/metadata_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/slice_tracker.h" #include "src/trace_processor/importers/common/stack_profile_tracker.h" @@ -259,12 +262,16 @@ class ProtoTraceParserTest : public ::testing::Test { context_.args_translation_table.reset(new ArgsTranslationTable(storage_)); context_.metadata_tracker.reset( new MetadataTracker(context_.storage.get())); + context_.machine_tracker.reset(new MachineTracker(&context_, 0)); + context_.cpu_tracker.reset(new CpuTracker(&context_)); event_ = new MockEventTracker(&context_); context_.event_tracker.reset(event_); sched_ = new MockSchedEventTracker(&context_); context_.ftrace_sched_tracker.reset(sched_); process_ = new NiceMock<MockProcessTracker>(&context_); context_.process_tracker.reset(process_); + context_.process_track_translation_table.reset( + new ProcessTrackTranslationTable(storage_)); slice_ = new NiceMock<MockSliceTracker>(&context_); context_.slice_tracker.reset(slice_); context_.slice_translation_table.reset(new SliceTranslationTable(storage_)); @@ -2368,7 +2375,9 @@ TEST_F(ProtoTraceParserTest, TrackEventParseLegacyEventIntoRawTable) { EXPECT_EQ(raw_table.ts()[0], 1010000); EXPECT_EQ(raw_table.name()[0], storage_->InternString("track_event.legacy_event")); - EXPECT_EQ(raw_table.cpu()[0], 0u); + auto ucpu = raw_table.ucpu()[0]; + const auto& cpu_table = storage_->cpu_table(); + EXPECT_EQ(cpu_table.cpu()[ucpu.value], 0u); EXPECT_EQ(raw_table.utid()[0], 1u); EXPECT_EQ(raw_table.arg_set_id()[0], 1u); diff --git a/src/trace_processor/importers/proto/proto_trace_reader.cc b/src/trace_processor/importers/proto/proto_trace_reader.cc index 2e50f0cc3..d958c0b6a 100644 --- a/src/trace_processor/importers/proto/proto_trace_reader.cc +++ b/src/trace_processor/importers/proto/proto_trace_reader.cc @@ -21,6 +21,7 @@ #include "perfetto/base/build_config.h" #include "perfetto/base/logging.h" +#include "perfetto/ext/base/flat_hash_map.h" #include "perfetto/ext/base/string_view.h" #include "perfetto/ext/base/utils.h" #include "perfetto/protozero/proto_decoder.h" @@ -41,6 +42,7 @@ #include "src/trace_processor/util/gzip_utils.h" #include "protos/perfetto/common/builtin_clock.pbzero.h" +#include "protos/perfetto/common/trace_stats.pbzero.h" #include "protos/perfetto/config/trace_config.pbzero.h" #include "protos/perfetto/trace/clock_snapshot.pbzero.h" #include "protos/perfetto/trace/extension_descriptor.pbzero.h" @@ -118,6 +120,16 @@ util::Status ProtoTraceReader::ParsePacket(TraceBlobView packet) { HandlePreviousPacketDropped(decoder); } + uint32_t sequence_id = decoder.trusted_packet_sequence_id(); + if (sequence_id) { + auto [data_loss, inserted] = + packet_sequence_data_loss_.Insert(sequence_id, 0); + + if (!inserted && decoder.previous_packet_dropped()) { + *data_loss += 1; + } + } + // It is important that we parse defaults before parsing other fields such as // the timestamp, since the defaults could affect them. if (decoder.has_trace_packet_defaults()) { @@ -131,8 +143,11 @@ util::Status ProtoTraceReader::ParsePacket(TraceBlobView packet) { } if (decoder.has_clock_snapshot()) { - return ParseClockSnapshot(decoder.clock_snapshot(), - decoder.trusted_packet_sequence_id()); + return ParseClockSnapshot(decoder.clock_snapshot(), sequence_id); + } + + if (decoder.has_trace_stats()) { + ParseTraceStats(decoder.trace_stats()); } if (decoder.has_service_event()) { @@ -474,6 +489,125 @@ util::Status ProtoTraceReader::ParseServiceEvent(int64_t ts, ConstBytes blob) { return util::OkStatus(); } +void ProtoTraceReader::ParseTraceStats(ConstBytes blob) { + protos::pbzero::TraceStats::Decoder evt(blob.data, blob.size); + auto* storage = context_->storage.get(); + storage->SetStats(stats::traced_producers_connected, + static_cast<int64_t>(evt.producers_connected())); + storage->SetStats(stats::traced_producers_seen, + static_cast<int64_t>(evt.producers_seen())); + storage->SetStats(stats::traced_data_sources_registered, + static_cast<int64_t>(evt.data_sources_registered())); + storage->SetStats(stats::traced_data_sources_seen, + static_cast<int64_t>(evt.data_sources_seen())); + storage->SetStats(stats::traced_tracing_sessions, + static_cast<int64_t>(evt.tracing_sessions())); + storage->SetStats(stats::traced_total_buffers, + static_cast<int64_t>(evt.total_buffers())); + storage->SetStats(stats::traced_chunks_discarded, + static_cast<int64_t>(evt.chunks_discarded())); + storage->SetStats(stats::traced_patches_discarded, + static_cast<int64_t>(evt.patches_discarded())); + storage->SetStats(stats::traced_flushes_requested, + static_cast<int64_t>(evt.flushes_requested())); + storage->SetStats(stats::traced_flushes_succeeded, + static_cast<int64_t>(evt.flushes_succeeded())); + storage->SetStats(stats::traced_flushes_failed, + static_cast<int64_t>(evt.flushes_failed())); + + if (evt.has_filter_stats()) { + protos::pbzero::TraceStats::FilterStats::Decoder fstat(evt.filter_stats()); + storage->SetStats(stats::filter_errors, + static_cast<int64_t>(fstat.errors())); + storage->SetStats(stats::filter_input_bytes, + static_cast<int64_t>(fstat.input_bytes())); + storage->SetStats(stats::filter_input_packets, + static_cast<int64_t>(fstat.input_packets())); + storage->SetStats(stats::filter_output_bytes, + static_cast<int64_t>(fstat.output_bytes())); + storage->SetStats(stats::filter_time_taken_ns, + static_cast<int64_t>(fstat.time_taken_ns())); + for (auto [i, it] = std::tuple{0, fstat.bytes_discarded_per_buffer()}; it; + ++it, ++i) { + storage->SetIndexedStats(stats::traced_buf_bytes_filtered_out, i, + static_cast<int64_t>(*it)); + } + } + + switch (evt.final_flush_outcome()) { + case protos::pbzero::TraceStats::FINAL_FLUSH_SUCCEEDED: + storage->IncrementStats(stats::traced_final_flush_succeeded, 1); + break; + case protos::pbzero::TraceStats::FINAL_FLUSH_FAILED: + storage->IncrementStats(stats::traced_final_flush_failed, 1); + break; + case protos::pbzero::TraceStats::FINAL_FLUSH_UNSPECIFIED: + break; + } + + int buf_num = 0; + for (auto it = evt.buffer_stats(); it; ++it, ++buf_num) { + protos::pbzero::TraceStats::BufferStats::Decoder buf(*it); + storage->SetIndexedStats(stats::traced_buf_buffer_size, buf_num, + static_cast<int64_t>(buf.buffer_size())); + storage->SetIndexedStats(stats::traced_buf_bytes_written, buf_num, + static_cast<int64_t>(buf.bytes_written())); + storage->SetIndexedStats(stats::traced_buf_bytes_overwritten, buf_num, + static_cast<int64_t>(buf.bytes_overwritten())); + storage->SetIndexedStats(stats::traced_buf_bytes_read, buf_num, + static_cast<int64_t>(buf.bytes_read())); + storage->SetIndexedStats(stats::traced_buf_padding_bytes_written, buf_num, + static_cast<int64_t>(buf.padding_bytes_written())); + storage->SetIndexedStats(stats::traced_buf_padding_bytes_cleared, buf_num, + static_cast<int64_t>(buf.padding_bytes_cleared())); + storage->SetIndexedStats(stats::traced_buf_chunks_written, buf_num, + static_cast<int64_t>(buf.chunks_written())); + storage->SetIndexedStats(stats::traced_buf_chunks_rewritten, buf_num, + static_cast<int64_t>(buf.chunks_rewritten())); + storage->SetIndexedStats(stats::traced_buf_chunks_overwritten, buf_num, + static_cast<int64_t>(buf.chunks_overwritten())); + storage->SetIndexedStats(stats::traced_buf_chunks_discarded, buf_num, + static_cast<int64_t>(buf.chunks_discarded())); + storage->SetIndexedStats(stats::traced_buf_chunks_read, buf_num, + static_cast<int64_t>(buf.chunks_read())); + storage->SetIndexedStats( + stats::traced_buf_chunks_committed_out_of_order, buf_num, + static_cast<int64_t>(buf.chunks_committed_out_of_order())); + storage->SetIndexedStats(stats::traced_buf_write_wrap_count, buf_num, + static_cast<int64_t>(buf.write_wrap_count())); + storage->SetIndexedStats(stats::traced_buf_patches_succeeded, buf_num, + static_cast<int64_t>(buf.patches_succeeded())); + storage->SetIndexedStats(stats::traced_buf_patches_failed, buf_num, + static_cast<int64_t>(buf.patches_failed())); + storage->SetIndexedStats(stats::traced_buf_readaheads_succeeded, buf_num, + static_cast<int64_t>(buf.readaheads_succeeded())); + storage->SetIndexedStats(stats::traced_buf_readaheads_failed, buf_num, + static_cast<int64_t>(buf.readaheads_failed())); + storage->SetIndexedStats(stats::traced_buf_abi_violations, buf_num, + static_cast<int64_t>(buf.abi_violations())); + storage->SetIndexedStats( + stats::traced_buf_trace_writer_packet_loss, buf_num, + static_cast<int64_t>(buf.trace_writer_packet_loss())); + } + + base::FlatHashMap<int32_t, int64_t> data_loss_per_buffer; + + for (auto it = evt.writer_stats(); it; ++it) { + protos::pbzero::TraceStats::WriterStats::Decoder writer(*it); + auto* data_loss = packet_sequence_data_loss_.Find( + static_cast<uint32_t>(writer.sequence_id())); + if (data_loss) { + data_loss_per_buffer[static_cast<int32_t>(writer.buffer())] += + static_cast<int64_t>(*data_loss); + } + } + + for (auto it = data_loss_per_buffer.GetIterator(); it; ++it) { + storage->SetIndexedStats(stats::traced_buf_sequence_packet_loss, it.key(), + it.value()); + } +} + void ProtoTraceReader::NotifyEndOfFile() {} } // namespace trace_processor diff --git a/src/trace_processor/importers/proto/proto_trace_reader.h b/src/trace_processor/importers/proto/proto_trace_reader.h index a93ad9881..4c4e2ca2d 100644 --- a/src/trace_processor/importers/proto/proto_trace_reader.h +++ b/src/trace_processor/importers/proto/proto_trace_reader.h @@ -78,6 +78,7 @@ class ProtoTraceReader : public ChunkedTraceReader { void ParseInternedData(const protos::pbzero::TracePacket_Decoder&, TraceBlobView interned_data); void ParseTraceConfig(ConstBytes); + void ParseTraceStats(ConstBytes); std::optional<StringId> GetBuiltinClockNameOrNull(int64_t clock_id); @@ -104,6 +105,8 @@ class ProtoTraceReader : public ChunkedTraceReader { base::FlatHashMap<uint32_t, PacketSequenceStateBuilder> packet_sequence_state_builders_; + base::FlatHashMap<uint32_t, size_t> packet_sequence_data_loss_; + StringId skipped_packet_key_id_; StringId invalid_incremental_state_key_id_; }; diff --git a/src/trace_processor/importers/proto/statsd_module.cc b/src/trace_processor/importers/proto/statsd_module.cc index e925b5a63..62faf0102 100644 --- a/src/trace_processor/importers/proto/statsd_module.cc +++ b/src/trace_processor/importers/proto/statsd_module.cc @@ -23,6 +23,7 @@ #include "src/trace_processor/importers/common/machine_tracker.h" #include "src/trace_processor/importers/common/slice_tracker.h" #include "src/trace_processor/importers/common/track_tracker.h" +#include "src/trace_processor/importers/proto/args_parser.h" #include "src/trace_processor/importers/proto/packet_sequence_state_generation.h" #include "src/trace_processor/sorter/trace_sorter.h" #include "src/trace_processor/storage/stats.h" @@ -37,111 +38,6 @@ namespace { constexpr const char* kAtomProtoName = ".android.os.statsd.Atom"; -using BoundInserter = ArgsTracker::BoundInserter; - -class InserterDelegate : public util::ProtoToArgsParser::Delegate { - public: - InserterDelegate(BoundInserter& inserter, TraceStorage& storage) - : inserter_(inserter), storage_(storage) {} - ~InserterDelegate() override = default; - - using Key = util::ProtoToArgsParser::Key; - - void AddInteger(const Key& key, int64_t value) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = Variadic::Integer(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - void AddUnsignedInteger(const Key& key, uint64_t value) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = Variadic::UnsignedInteger(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - void AddString(const Key& key, const protozero::ConstChars& value) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = Variadic::String(storage_.InternString(value)); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - void AddString(const Key& key, const std::string& value) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = - Variadic::String(storage_.InternString(base::StringView(value))); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - void AddDouble(const Key& key, double value) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = Variadic::Real(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - void AddPointer(const Key& key, const void* value) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = - Variadic::Pointer(reinterpret_cast<uintptr_t>(value)); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - void AddBoolean(const Key& key, bool value) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = Variadic::Boolean(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - bool AddJson(const Key&, const protozero::ConstChars&) override { - PERFETTO_FATAL("Unexpected JSON value when parsing statsd data"); - } - - void AddNull(const Key& key) override { - StringId flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - StringId key_id = storage_.InternString(base::StringView(key.key)); - Variadic variadic_val = Variadic::Null(); - inserter_.AddArg(flat_key_id, key_id, variadic_val); - } - - size_t GetArrayEntryIndex(const std::string& array_key) override { - base::ignore_result(array_key); - return 0; - } - - size_t IncrementArrayEntryIndex(const std::string& array_key) override { - base::ignore_result(array_key); - return 0; - } - - PacketSequenceStateGeneration* seq_state() override { return nullptr; } - - protected: - InternedMessageView* GetInternedMessageView(uint32_t field_id, - uint64_t iid) override { - base::ignore_result(field_id); - base::ignore_result(iid); - return nullptr; - } - - private: - BoundInserter& inserter_; - TraceStorage& storage_; -}; - // If we don't know about the atom format put whatever details we // can. This has the following restrictions: // - We can't tell the difference between double, fixed64, sfixed64 @@ -296,7 +192,7 @@ void StatsdModule::ParseAtom(int64_t ts, protozero::ConstBytes nested_bytes) { } SliceId slice = opt_slice.value(); auto inserter = context_->args_tracker->AddArgsTo(slice); - InserterDelegate delegate(inserter, *context_->storage.get()); + ArgsParser delegate(ts, inserter, *context_->storage.get()); const auto& fields = pool_.descriptor()->fields(); const auto& field_it = fields.find(nested_field_id); diff --git a/src/trace_processor/importers/proto/system_probes_parser.cc b/src/trace_processor/importers/proto/system_probes_parser.cc index 5b878eddd..1ebc13be7 100644 --- a/src/trace_processor/importers/proto/system_probes_parser.cc +++ b/src/trace_processor/importers/proto/system_probes_parser.cc @@ -22,6 +22,7 @@ #include "perfetto/ext/traced/sys_stats_counters.h" #include "perfetto/protozero/proto_decoder.h" #include "src/trace_processor/importers/common/clock_tracker.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/event_tracker.h" #include "src/trace_processor/importers/common/metadata_tracker.h" #include "src/trace_processor/importers/common/process_tracker.h" @@ -168,6 +169,8 @@ SystemProbesParser::SystemProbesParser(TraceProcessorContext* context) context->storage->InternString("mem.smaps.pss.file"); proc_stats_process_names_[ProcessStats::Process::kSmrPssShmemKbFieldNumber] = context->storage->InternString("mem.smaps.pss.shmem"); + proc_stats_process_names_[ProcessStats::Process::kSmrSwapPssKbFieldNumber] = + context->storage->InternString("mem.smaps.swap.pss"); proc_stats_process_names_ [ProcessStats::Process::kRuntimeUserModeFieldNumber] = context->storage->InternString("runtime.user_ns"); @@ -744,13 +747,10 @@ void SystemProbesParser::ParseSystemInfo(ConstBytes blob) { void SystemProbesParser::ParseCpuInfo(ConstBytes blob) { protos::pbzero::CpuInfo::Decoder packet(blob.data, blob.size); uint32_t cluster_id = 0; + uint32_t cpu_id = 0; std::vector<uint32_t> last_cpu_freqs; - for (auto it = packet.cpus(); it; it++) { + for (auto it = packet.cpus(); it; it++, cpu_id++) { protos::pbzero::CpuInfo::Cpu::Decoder cpu(*it); - tables::CpuTable::Row cpu_row; - if (cpu.has_processor()) { - cpu_row.processor = context_->storage->InternString(cpu.processor()); - } std::vector<uint32_t> freqs; for (auto freq_it = cpu.frequencies(); freq_it; freq_it++) { freqs.push_back(*freq_it); @@ -760,19 +760,17 @@ void SystemProbesParser::ParseCpuInfo(ConstBytes blob) { if (freqs != last_cpu_freqs && !last_cpu_freqs.empty()) { cluster_id++; } - cpu_row.cluster_id = cluster_id; - cpu_row.machine_id = context_->machine_id(); last_cpu_freqs = freqs; - tables::CpuTable::Id cpu_row_id = - context_->storage->mutable_cpu_table()->Insert(cpu_row).id; + + tables::CpuTable::Id ucpu = + context_->cpu_tracker->SetCpuInfo(cpu_id, cpu.processor(), cluster_id); for (auto freq_it = cpu.frequencies(); freq_it; freq_it++) { uint32_t freq = *freq_it; tables::CpuFreqTable::Row cpu_freq_row; - cpu_freq_row.cpu_id = cpu_row_id; + cpu_freq_row.ucpu = ucpu; cpu_freq_row.freq = freq; - cpu_freq_row.machine_id = context_->machine_id(); context_->storage->mutable_cpu_freq_table()->Insert(cpu_freq_row); } } diff --git a/src/trace_processor/importers/proto/system_probes_parser.h b/src/trace_processor/importers/proto/system_probes_parser.h index a54beb7b3..472e334ee 100644 --- a/src/trace_processor/importers/proto/system_probes_parser.h +++ b/src/trace_processor/importers/proto/system_probes_parser.h @@ -75,7 +75,7 @@ class SystemProbesParser { // their StringId. Keep kProcStatsProcessSize equal to 1 + max proto field // id of ProcessStats::Process. Also update the value in // ChromeSystemProbesParser. - static constexpr size_t kProcStatsProcessSize = 23; + static constexpr size_t kProcStatsProcessSize = 24; std::array<StringId, kProcStatsProcessSize> proc_stats_process_names_{}; // Maps a SysStats::PsiSample::PsiResource type to its StringId. diff --git a/src/trace_processor/importers/proto/track_event_parser.cc b/src/trace_processor/importers/proto/track_event_parser.cc index 585463049..e252be5e1 100644 --- a/src/trace_processor/importers/proto/track_event_parser.cc +++ b/src/trace_processor/importers/proto/track_event_parser.cc @@ -26,13 +26,15 @@ #include "perfetto/trace_processor/status.h" #include "src/trace_processor/importers/common/args_tracker.h" #include "src/trace_processor/importers/common/args_translation_table.h" +#include "src/trace_processor/importers/common/cpu_tracker.h" #include "src/trace_processor/importers/common/event_tracker.h" #include "src/trace_processor/importers/common/flow_tracker.h" -#include "src/trace_processor/importers/common/machine_tracker.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/importers/common/virtual_memory_mapping.h" #include "src/trace_processor/importers/json/json_utils.h" +#include "src/trace_processor/importers/proto/args_parser.h" #include "src/trace_processor/importers/proto/packet_analyzer.h" #include "src/trace_processor/importers/proto/profile_packet_utils.h" #include "src/trace_processor/importers/proto/stack_profile_sequence_state.h" @@ -74,103 +76,6 @@ using protozero::ConstBytes; constexpr int64_t kPendingThreadDuration = -1; constexpr int64_t kPendingThreadInstructionDelta = -1; -class TrackEventArgsParser : public util::ProtoToArgsParser::Delegate { - public: - TrackEventArgsParser(int64_t packet_timestamp, - BoundInserter& inserter, - TraceStorage& storage, - PacketSequenceStateGeneration& sequence_state) - : packet_timestamp_(packet_timestamp), - inserter_(inserter), - storage_(storage), - sequence_state_(sequence_state) {} - - ~TrackEventArgsParser() override; - - using Key = util::ProtoToArgsParser::Key; - - void AddInteger(const Key& key, int64_t value) final { - inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::Integer(value)); - } - void AddUnsignedInteger(const Key& key, uint64_t value) final { - inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::UnsignedInteger(value)); - } - void AddString(const Key& key, const protozero::ConstChars& value) final { - inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::String(storage_.InternString(value))); - } - void AddString(const Key& key, const std::string& value) final { - inserter_.AddArg( - storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::String(storage_.InternString(base::StringView(value)))); - } - void AddDouble(const Key& key, double value) final { - inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::Real(value)); - } - void AddPointer(const Key& key, const void* value) final { - inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::Pointer(reinterpret_cast<uintptr_t>(value))); - } - void AddBoolean(const Key& key, bool value) final { - inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::Boolean(value)); - } - void AddBytes(const Key& key, const protozero::ConstBytes& value) final { - std::string b64_data = base::Base64Encode(value.data, value.size); - AddString(key, b64_data); - } - bool AddJson(const Key& key, const protozero::ConstChars& value) final { - auto json_value = json::ParseJsonString(value); - if (!json_value) - return false; - return json::AddJsonValueToArgs(*json_value, base::StringView(key.flat_key), - base::StringView(key.key), &storage_, - &inserter_); - } - void AddNull(const Key& key) final { - inserter_.AddArg(storage_.InternString(base::StringView(key.flat_key)), - storage_.InternString(base::StringView(key.key)), - Variadic::Null()); - } - - size_t GetArrayEntryIndex(const std::string& array_key) final { - return inserter_.GetNextArrayEntryIndex( - storage_.InternString(base::StringView(array_key))); - } - - size_t IncrementArrayEntryIndex(const std::string& array_key) final { - return inserter_.IncrementArrayEntryIndex( - storage_.InternString(base::StringView(array_key))); - } - - InternedMessageView* GetInternedMessageView(uint32_t field_id, - uint64_t iid) final { - return sequence_state_.GetInternedMessageView(field_id, iid); - } - - int64_t packet_timestamp() final { return packet_timestamp_; } - - PacketSequenceStateGeneration* seq_state() final { return &sequence_state_; } - - private: - int64_t packet_timestamp_; - BoundInserter& inserter_; - TraceStorage& storage_; - PacketSequenceStateGeneration& sequence_state_; -}; - -TrackEventArgsParser::~TrackEventArgsParser() = default; - // Paths on Windows use backslash rather than slash as a separator. // Normalise the paths by replacing backslashes with slashes to make it // easier to write cross-platform scripts. @@ -1118,10 +1023,11 @@ class TrackEventParser::EventImporter { if (!utid_) return util::ErrStatus("raw legacy event without thread association"); - RawId id = storage_->mutable_raw_table() - ->Insert({ts_, parser_->raw_legacy_event_id_, 0, *utid_, 0, - 0, context_->machine_id()}) - .id; + auto ucpu = context_->cpu_tracker->GetOrCreateCpu(0); + RawId id = + storage_->mutable_raw_table() + ->Insert({ts_, parser_->raw_legacy_event_id_, *utid_, 0, 0, ucpu}) + .id; auto inserter = context_->args_tracker->AddArgsTo(id); inserter @@ -1226,8 +1132,8 @@ class TrackEventParser::EventImporter { } } - TrackEventArgsParser args_writer(ts_, *inserter, *storage_, - *sequence_state_); + ArgsParser args_writer(ts_, *inserter, *storage_, sequence_state_, + /*support_json=*/true); int unknown_extensions = 0; log_errors(parser_->args_parser_.ParseMessage( blob_, ".perfetto.protos.TrackEvent", &parser_->reflect_fields_, @@ -1624,8 +1530,10 @@ void TrackEventParser::ParseTrackDescriptor( // Override the name with the most recent name seen (after sorting by ts). if (decoder.has_name() || decoder.has_static_name()) { auto* tracks = context_->storage->mutable_track_table(); - StringId name_id = context_->storage->InternString( + const StringId raw_name_id = context_->storage->InternString( decoder.has_name() ? decoder.name() : decoder.static_name()); + const StringId name_id = + context_->process_track_translation_table->TranslateName(raw_name_id); tracks->mutable_name()->Set(*tracks->id().IndexOf(track_id), name_id); } } diff --git a/src/trace_processor/importers/proto/track_event_tracker.cc b/src/trace_processor/importers/proto/track_event_tracker.cc index 56e081dd6..e758b9d05 100644 --- a/src/trace_processor/importers/proto/track_event_tracker.cc +++ b/src/trace_processor/importers/proto/track_event_tracker.cc @@ -18,6 +18,7 @@ #include "src/trace_processor/importers/common/args_tracker.h" #include "src/trace_processor/importers/common/args_translation_table.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/process_tracker.h" #include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/tables/track_tables_py.h" @@ -201,7 +202,9 @@ std::optional<TrackId> TrackEventTracker::GetDescriptorTrack( reservation_it->second.is_counter) { return track_id; } - tracks->mutable_name()->Set(row, event_name); + const StringId track_name = + context_->process_track_translation_table->TranslateName(event_name); + tracks->mutable_name()->Set(row, track_name); return track_id; } diff --git a/src/trace_processor/importers/proto/translation_table_module.cc b/src/trace_processor/importers/proto/translation_table_module.cc index 8684df031..f45e35326 100644 --- a/src/trace_processor/importers/proto/translation_table_module.cc +++ b/src/trace_processor/importers/proto/translation_table_module.cc @@ -16,6 +16,7 @@ #include "src/trace_processor/importers/proto/translation_table_module.h" #include "src/trace_processor/importers/common/args_translation_table.h" +#include "src/trace_processor/importers/common/process_track_translation_table.h" #include "src/trace_processor/importers/common/slice_translation_table.h" #include "src/trace_processor/importers/proto/packet_sequence_state_generation.h" @@ -54,6 +55,8 @@ ModuleResult TranslationTableModule::TokenizePacket( translation_table.chrome_performance_mark()); } else if (translation_table.has_slice_name()) { ParseSliceNameRules(translation_table.slice_name()); + } else if (translation_table.has_process_track_name()) { + ParseProcessTrackNameRules(translation_table.process_track_name()); } return ModuleResult::Handled(); } @@ -113,5 +116,17 @@ void TranslationTableModule::ParseSliceNameRules(protozero::ConstBytes bytes) { } } +void TranslationTableModule::ParseProcessTrackNameRules( + protozero::ConstBytes bytes) { + const auto process_track_name = + protos::pbzero::ProcessTrackNameTranslationTable::Decoder(bytes); + for (auto it = process_track_name.raw_to_deobfuscated_name(); it; ++it) { + protos::pbzero::ProcessTrackNameTranslationTable:: + RawToDeobfuscatedNameEntry::Decoder entry(*it); + context_->process_track_translation_table->AddNameTranslationRule( + entry.key(), entry.value()); + } +} + } // namespace trace_processor } // namespace perfetto diff --git a/src/trace_processor/importers/proto/translation_table_module.h b/src/trace_processor/importers/proto/translation_table_module.h index 2060f4139..62d4c725b 100644 --- a/src/trace_processor/importers/proto/translation_table_module.h +++ b/src/trace_processor/importers/proto/translation_table_module.h @@ -46,6 +46,7 @@ class TranslationTableModule : public ProtoImporterModule { void ParseChromeUserEventRules(protozero::ConstBytes bytes); void ParseChromePerformanceMarkRules(protozero::ConstBytes bytes); void ParseSliceNameRules(protozero::ConstBytes bytes); + void ParseProcessTrackNameRules(protozero::ConstBytes bytes); TraceProcessorContext* context_; }; diff --git a/src/trace_processor/importers/proto/winscope/BUILD.gn b/src/trace_processor/importers/proto/winscope/BUILD.gn index 09b7b77b3..f518f424a 100644 --- a/src/trace_processor/importers/proto/winscope/BUILD.gn +++ b/src/trace_processor/importers/proto/winscope/BUILD.gn @@ -28,8 +28,8 @@ source_set("full") { "surfaceflinger_layers_parser.h", "surfaceflinger_transactions_parser.cc", "surfaceflinger_transactions_parser.h", - "winscope_args_parser.cc", - "winscope_args_parser.h", + "viewcapture_args_parser.cc", + "viewcapture_args_parser.h", "winscope_module.cc", "winscope_module.h", ] @@ -38,10 +38,10 @@ source_set("full") { "../:proto_importer_module", "../../../../../gn:default_deps", "../../../../../protos/perfetto/trace:zero", - "../../../../../protos/perfetto/trace/android:zero", "../../../../../protos/perfetto/trace/android:winscope_common_zero", "../../../../../protos/perfetto/trace/android:winscope_extensions_zero", "../../../../../protos/perfetto/trace/android:winscope_regular_zero", + "../../../../../protos/perfetto/trace/android:zero", "../../../../../protos/perfetto/trace/interned_data:zero", "../../../../../protos/perfetto/trace/profiling:zero", "../../../../protozero", diff --git a/src/trace_processor/importers/proto/winscope/shell_transitions_parser.cc b/src/trace_processor/importers/proto/winscope/shell_transitions_parser.cc index 68d2a82c6..12618acdc 100644 --- a/src/trace_processor/importers/proto/winscope/shell_transitions_parser.cc +++ b/src/trace_processor/importers/proto/winscope/shell_transitions_parser.cc @@ -19,8 +19,8 @@ #include "protos/perfetto/trace/android/shell_transition.pbzero.h" #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/proto/args_parser.h" #include "src/trace_processor/importers/proto/winscope/winscope.descriptor.h" -#include "src/trace_processor/importers/proto/winscope/winscope_args_parser.h" #include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/types/trace_processor_context.h" @@ -49,7 +49,7 @@ void ShellTransitionsParser::ParseTransition(protozero::ConstBytes blob) { } auto inserter = context_->args_tracker->AddArgsTo(row_id); - WinscopeArgsParser writer(inserter, *context_->storage.get()); + ArgsParser writer(/*timestamp=*/0, inserter, *context_->storage.get()); base::Status status = args_parser_.ParseMessage( blob, kShellTransitionsProtoName, nullptr /* parse all fields */, writer); diff --git a/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.cc b/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.cc index ebe370c6d..20687e4e1 100644 --- a/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.cc +++ b/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.cc @@ -18,8 +18,8 @@ #include "protos/perfetto/trace/android/surfaceflinger_layers.pbzero.h" #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/proto/args_parser.h" #include "src/trace_processor/importers/proto/winscope/winscope.descriptor.h" -#include "src/trace_processor/importers/proto/winscope/winscope_args_parser.h" #include "src/trace_processor/types/trace_processor_context.h" namespace perfetto { @@ -44,7 +44,7 @@ void SurfaceFlingerLayersParser::Parse(int64_t timestamp, .id; auto inserter = context_->args_tracker->AddArgsTo(snapshot_id); - WinscopeArgsParser writer(inserter, *context_->storage.get()); + ArgsParser writer(timestamp, inserter, *context_->storage); base::Status status = args_parser_.ParseMessage(blob, kLayersSnapshotProtoName, &kLayersSnapshotFieldsToArgsParse, writer); @@ -55,11 +55,12 @@ void SurfaceFlingerLayersParser::Parse(int64_t timestamp, protos::pbzero::LayersProto::Decoder layers_decoder( snapshot_decoder.layers().data, snapshot_decoder.layers().size); for (auto it = layers_decoder.layers(); it; ++it) { - ParseLayer(*it, snapshot_id); + ParseLayer(timestamp, *it, snapshot_id); } } void SurfaceFlingerLayersParser::ParseLayer( + int64_t timestamp, protozero::ConstBytes blob, tables::SurfaceFlingerLayersSnapshotTable::Id snapshot_id) { tables::SurfaceFlingerLayerTable::Row layer; @@ -69,7 +70,7 @@ void SurfaceFlingerLayersParser::ParseLayer( ArgsTracker tracker(context_); auto inserter = tracker.AddArgsTo(layerId); - WinscopeArgsParser writer(inserter, *context_->storage.get()); + ArgsParser writer(timestamp, inserter, *context_->storage); base::Status status = args_parser_.ParseMessage( blob, kLayerProtoName, nullptr /* parse all fields */, writer); if (!status.ok()) { diff --git a/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.h b/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.h index 866cbae68..f615a8e42 100644 --- a/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.h +++ b/src/trace_processor/importers/proto/winscope/surfaceflinger_layers_parser.h @@ -39,7 +39,8 @@ class SurfaceFlingerLayersParser { ".perfetto.protos.LayersSnapshotProto"; static constexpr auto* kLayerProtoName = ".perfetto.protos.LayerProto"; - void ParseLayer(protozero::ConstBytes blob, + void ParseLayer(int64_t timestamp, + protozero::ConstBytes blob, tables::SurfaceFlingerLayersSnapshotTable::Id); TraceProcessorContext* const context_; diff --git a/src/trace_processor/importers/proto/winscope/surfaceflinger_transactions_parser.cc b/src/trace_processor/importers/proto/winscope/surfaceflinger_transactions_parser.cc index ae3a3e912..c2c563813 100644 --- a/src/trace_processor/importers/proto/winscope/surfaceflinger_transactions_parser.cc +++ b/src/trace_processor/importers/proto/winscope/surfaceflinger_transactions_parser.cc @@ -18,8 +18,8 @@ #include "protos/perfetto/trace/android/surfaceflinger_transactions.pbzero.h" #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/proto/args_parser.h" #include "src/trace_processor/importers/proto/winscope/winscope.descriptor.h" -#include "src/trace_processor/importers/proto/winscope/winscope_args_parser.h" #include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/types/trace_processor_context.h" @@ -43,7 +43,7 @@ void SurfaceFlingerTransactionsParser::Parse(int64_t timestamp, ArgsTracker tracker(context_); auto inserter = tracker.AddArgsTo(rowId); - WinscopeArgsParser writer(inserter, *context_->storage.get()); + ArgsParser writer(timestamp, inserter, *context_->storage.get()); base::Status status = args_parser_.ParseMessage(blob, kTransactionTraceEntryProtoName, nullptr /* parse all fields */, writer); diff --git a/src/trace_processor/importers/proto/winscope/viewcapture_args_parser.cc b/src/trace_processor/importers/proto/winscope/viewcapture_args_parser.cc new file mode 100644 index 000000000..ed881632e --- /dev/null +++ b/src/trace_processor/importers/proto/winscope/viewcapture_args_parser.cc @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/proto/winscope/viewcapture_args_parser.h" +#include "perfetto/ext/base/string_utils.h" +#include "perfetto/ext/base/string_view.h" +#include "protos/perfetto/trace/interned_data/interned_data.pbzero.h" +#include "protos/perfetto/trace/profiling/profile_common.pbzero.h" +#include "src/trace_processor/importers/proto/packet_sequence_state_generation.h" + +namespace perfetto { +namespace trace_processor { + +ViewCaptureArgsParser::ViewCaptureArgsParser( + int64_t packet_timestamp, + ArgsTracker::BoundInserter& inserter, + TraceStorage& storage, + PacketSequenceStateGeneration* sequence_state) + : ArgsParser(packet_timestamp, inserter, storage, sequence_state), + storage_{storage} {} + +void ViewCaptureArgsParser::AddInteger(const Key& key, int64_t value) { + if (TryAddDeinternedString(key, static_cast<uint64_t>(value))) { + return; + } + ArgsParser::AddInteger(key, value); +} + +void ViewCaptureArgsParser::AddUnsignedInteger(const Key& key, uint64_t value) { + if (TryAddDeinternedString(key, value)) { + return; + } + ArgsParser::AddUnsignedInteger(key, value); +} + +bool ViewCaptureArgsParser::TryAddDeinternedString(const Key& key, + uint64_t iid) { + bool is_interned_field = base::EndsWith(key.key, "_iid"); + if (!is_interned_field) { + return false; + } + + const auto deintern_key = key.key.substr(0, key.key.size() - 4); + const auto deintern_flat_key = + key.flat_key.substr(0, key.flat_key.size() - 4); + const auto deintern_key_combined = Key{deintern_flat_key, deintern_key}; + const auto deintern_val = TryDeinternString(key, iid); + + if (!deintern_val) { + ArgsParser::AddString( + deintern_key_combined, + protozero::ConstChars{ERROR_MSG.data(), ERROR_MSG.size()}); + storage_.IncrementStats( + stats::winscope_viewcapture_missing_interned_string_parse_errors); + return false; + } + + ArgsParser::AddString(deintern_key_combined, *deintern_val); + return true; +} + +std::optional<protozero::ConstChars> ViewCaptureArgsParser::TryDeinternString( + const Key& key, + uint64_t iid) { + if (base::EndsWith(key.key, "class_name_iid")) { + auto* decoder = + seq_state() + ->LookupInternedMessage< + protos::pbzero::InternedData::kViewcaptureClassNameFieldNumber, + protos::pbzero::InternedString>(iid); + if (decoder) { + return protozero::ConstChars{ + reinterpret_cast<const char*>(decoder->str().data), + decoder->str().size}; + } + } else if (base::EndsWith(key.key, "package_name_iid")) { + auto* decoder = + seq_state() + ->LookupInternedMessage<protos::pbzero::InternedData:: + kViewcapturePackageNameFieldNumber, + protos::pbzero::InternedString>(iid); + if (decoder) { + return protozero::ConstChars{ + reinterpret_cast<const char*>(decoder->str().data), + decoder->str().size}; + } + } else if (base::EndsWith(key.key, "view_id_iid")) { + auto* decoder = + seq_state() + ->LookupInternedMessage< + protos::pbzero::InternedData::kViewcaptureViewIdFieldNumber, + protos::pbzero::InternedString>(iid); + if (decoder) { + return protozero::ConstChars{ + reinterpret_cast<const char*>(decoder->str().data), + decoder->str().size}; + } + } else if (base::EndsWith(key.key, "window_name_iid")) { + auto* decoder = + seq_state() + ->LookupInternedMessage< + protos::pbzero::InternedData::kViewcaptureWindowNameFieldNumber, + protos::pbzero::InternedString>(iid); + if (decoder) { + return protozero::ConstChars{ + reinterpret_cast<const char*>(decoder->str().data), + decoder->str().size}; + } + } + + return std::nullopt; +} + +} // namespace trace_processor +} // namespace perfetto diff --git a/src/trace_processor/importers/proto/winscope/viewcapture_args_parser.h b/src/trace_processor/importers/proto/winscope/viewcapture_args_parser.h new file mode 100644 index 000000000..32b76b6e6 --- /dev/null +++ b/src/trace_processor/importers/proto/winscope/viewcapture_args_parser.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_WINSCOPE_VIEWCAPTURE_ARGS_PARSER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_WINSCOPE_VIEWCAPTURE_ARGS_PARSER_H_ + +#include <optional> + +#include "src/trace_processor/importers/proto/args_parser.h" + +namespace perfetto { +namespace trace_processor { + +// Specialized args parser to de-intern ViewCapture strings +class ViewCaptureArgsParser : public ArgsParser { + public: + using Key = ArgsParser::Key; + + ViewCaptureArgsParser(int64_t packet_timestamp, + ArgsTracker::BoundInserter& inserter, + TraceStorage& storage, + PacketSequenceStateGeneration* sequence_state); + void AddInteger(const Key&, int64_t) override; + void AddUnsignedInteger(const Key&, uint64_t) override; + + private: + bool TryAddDeinternedString(const Key&, uint64_t); + std::optional<protozero::ConstChars> TryDeinternString(const Key&, uint64_t); + + const base::StringView ERROR_MSG{"STRING DE-INTERNING ERROR"}; + TraceStorage& storage_; +}; + +} // namespace trace_processor +} // namespace perfetto + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_WINSCOPE_VIEWCAPTURE_ARGS_PARSER_H_ diff --git a/src/trace_processor/importers/proto/winscope/winscope_args_parser.cc b/src/trace_processor/importers/proto/winscope/winscope_args_parser.cc deleted file mode 100644 index aa4ff3e69..000000000 --- a/src/trace_processor/importers/proto/winscope/winscope_args_parser.cc +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/trace_processor/importers/proto/winscope/winscope_args_parser.h" - -namespace perfetto { -namespace trace_processor { - -WinscopeArgsParser::WinscopeArgsParser(ArgsTracker::BoundInserter& inserter, - TraceStorage& storage) - : inserter_{inserter}, storage_{storage} {} - -void WinscopeArgsParser::AddInteger(const Key& key, int64_t value) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = Variadic::Integer(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -void WinscopeArgsParser::AddUnsignedInteger(const Key& key, uint64_t value) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = Variadic::UnsignedInteger(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -void WinscopeArgsParser::AddString(const Key& key, - const protozero::ConstChars& value) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = Variadic::String(storage_.InternString(value)); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -void WinscopeArgsParser::AddString(const Key& key, const std::string& value) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = - Variadic::String(storage_.InternString(base::StringView(value))); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -void WinscopeArgsParser::AddDouble(const Key& key, double value) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = Variadic::Real(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -void WinscopeArgsParser::AddPointer(const Key& key, const void* value) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = - Variadic::Pointer(reinterpret_cast<uintptr_t>(value)); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -void WinscopeArgsParser::AddBoolean(const Key& key, bool value) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = Variadic::Boolean(value); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -bool WinscopeArgsParser::AddJson(const Key&, const protozero::ConstChars&) { - PERFETTO_FATAL("Unexpected JSON value when parsing SurfaceFlinger data"); -} - -void WinscopeArgsParser::AddNull(const Key& key) { - const auto flat_key_id = - storage_.InternString(base::StringView(key.flat_key)); - const auto key_id = storage_.InternString(base::StringView(key.key)); - const auto variadic_val = Variadic::Null(); - inserter_.AddArg(flat_key_id, key_id, variadic_val); -} - -size_t WinscopeArgsParser::GetArrayEntryIndex(const std::string& array_key) { - return inserter_.GetNextArrayEntryIndex( - storage_.InternString(base::StringView(array_key))); -} - -size_t WinscopeArgsParser::IncrementArrayEntryIndex( - const std::string& array_key) { - return inserter_.IncrementArrayEntryIndex( - storage_.InternString(base::StringView(array_key))); -} - -PacketSequenceStateGeneration* WinscopeArgsParser::seq_state() { - return nullptr; -} - -InternedMessageView* WinscopeArgsParser::GetInternedMessageView( - uint32_t field_id, - uint64_t iid) { - base::ignore_result(field_id); - base::ignore_result(iid); - return nullptr; -} - -} // namespace trace_processor -} // namespace perfetto diff --git a/src/trace_processor/importers/proto/winscope/winscope_module.cc b/src/trace_processor/importers/proto/winscope/winscope_module.cc index 8d7e03753..81c7a5953 100644 --- a/src/trace_processor/importers/proto/winscope/winscope_module.cc +++ b/src/trace_processor/importers/proto/winscope/winscope_module.cc @@ -17,8 +17,9 @@ #include "src/trace_processor/importers/proto/winscope/winscope_module.h" #include "protos/perfetto/trace/android/winscope_extensions.pbzero.h" #include "protos/perfetto/trace/android/winscope_extensions_impl.pbzero.h" +#include "src/trace_processor/importers/proto/args_parser.h" +#include "src/trace_processor/importers/proto/winscope/viewcapture_args_parser.h" #include "src/trace_processor/importers/proto/winscope/winscope.descriptor.h" -#include "src/trace_processor/importers/proto/winscope/winscope_args_parser.h" namespace perfetto { namespace trace_processor { @@ -76,13 +77,15 @@ void WinscopeModule::ParseTracePacketData(const TracePacket::Decoder& decoder, decoder.protolog_viewer_config()); return; case TracePacket::kWinscopeExtensionsFieldNumber: - ParseWinscopeExtensionsData(decoder.winscope_extensions(), timestamp); + ParseWinscopeExtensionsData(decoder.winscope_extensions(), timestamp, + data); return; } } void WinscopeModule::ParseWinscopeExtensionsData(protozero::ConstBytes blob, - int64_t timestamp) { + int64_t timestamp, + const TracePacketData& data) { WinscopeExtensionsImpl::Decoder decoder(blob.data, blob.size); if (auto field = @@ -97,6 +100,11 @@ void WinscopeModule::ParseWinscopeExtensionsData(protozero::ConstBytes blob, WinscopeExtensionsImpl::kInputmethodServiceFieldNumber); field.valid()) { ParseInputMethodServiceData(timestamp, field.as_bytes()); + } else if (field = + decoder.Get(WinscopeExtensionsImpl::kViewcaptureFieldNumber); + field.valid()) { + ParseViewCaptureData(timestamp, field.as_bytes(), + data.sequence_state.get()); } } @@ -109,7 +117,7 @@ void WinscopeModule::ParseInputMethodClientsData(int64_t timestamp, ArgsTracker tracker(context_); auto inserter = tracker.AddArgsTo(rowId); - WinscopeArgsParser writer(inserter, *context_->storage.get()); + ArgsParser writer(timestamp, inserter, *context_->storage.get()); base::Status status = args_parser_.ParseMessage(blob, kInputMethodClientsProtoName, nullptr /* parse all fields */, writer); @@ -130,7 +138,7 @@ void WinscopeModule::ParseInputMethodManagerServiceData( ArgsTracker tracker(context_); auto inserter = tracker.AddArgsTo(rowId); - WinscopeArgsParser writer(inserter, *context_->storage.get()); + ArgsParser writer(timestamp, inserter, *context_->storage.get()); base::Status status = args_parser_.ParseMessage(blob, kInputMethodManagerServiceProtoName, nullptr /* parse all fields */, writer); @@ -149,7 +157,7 @@ void WinscopeModule::ParseInputMethodServiceData(int64_t timestamp, ArgsTracker tracker(context_); auto inserter = tracker.AddArgsTo(rowId); - WinscopeArgsParser writer(inserter, *context_->storage.get()); + ArgsParser writer(timestamp, inserter, *context_->storage.get()); base::Status status = args_parser_.ParseMessage(blob, kInputMethodServiceProtoName, nullptr /* parse all fields */, writer); @@ -159,5 +167,24 @@ void WinscopeModule::ParseInputMethodServiceData(int64_t timestamp, } } +void WinscopeModule::ParseViewCaptureData( + int64_t timestamp, + protozero::ConstBytes blob, + PacketSequenceStateGeneration* sequence_state) { + tables::ViewCaptureTable::Row row; + row.ts = timestamp; + auto rowId = context_->storage->mutable_viewcapture_table()->Insert(row).id; + + ArgsTracker tracker(context_); + auto inserter = tracker.AddArgsTo(rowId); + ViewCaptureArgsParser writer(timestamp, inserter, *context_->storage.get(), + sequence_state); + base::Status status = args_parser_.ParseMessage( + blob, kViewCaptureProtoName, nullptr /* parse all fields */, writer); + if (!status.ok()) { + context_->storage->IncrementStats(stats::winscope_viewcapture_parse_errors); + } +} + } // namespace trace_processor } // namespace perfetto diff --git a/src/trace_processor/importers/proto/winscope/winscope_module.h b/src/trace_processor/importers/proto/winscope/winscope_module.h index c77fcb49c..7f6ae083c 100644 --- a/src/trace_processor/importers/proto/winscope/winscope_module.h +++ b/src/trace_processor/importers/proto/winscope/winscope_module.h @@ -42,21 +42,25 @@ class WinscopeModule : public ProtoImporterModule { private: void ParseWinscopeExtensionsData(protozero::ConstBytes blob, - int64_t timestamp); + int64_t timestamp, + const TracePacketData&); void ParseInputMethodClientsData(int64_t timestamp, protozero::ConstBytes blob); void ParseInputMethodManagerServiceData(int64_t timestamp, protozero::ConstBytes blob); void ParseInputMethodServiceData(int64_t timestamp, protozero::ConstBytes blob); + void ParseViewCaptureData(int64_t timestamp, + protozero::ConstBytes blob, + PacketSequenceStateGeneration* sequence_state); static constexpr auto* kInputMethodClientsProtoName = ".perfetto.protos.InputMethodClientsTraceProto"; static constexpr auto* kInputMethodManagerServiceProtoName = ".perfetto.protos.InputMethodManagerServiceTraceProto"; - static constexpr auto* kInputMethodServiceProtoName = ".perfetto.protos.InputMethodServiceTraceProto"; + static constexpr auto* kViewCaptureProtoName = ".perfetto.protos.ViewCapture"; TraceProcessorContext* const context_; DescriptorPool pool_; diff --git a/src/trace_processor/importers/zip/BUILD.gn b/src/trace_processor/importers/zip/BUILD.gn new file mode 100644 index 000000000..0262e02cd --- /dev/null +++ b/src/trace_processor/importers/zip/BUILD.gn @@ -0,0 +1,32 @@ +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source_set("full") { + sources = [ + "zip_trace_reader.cc", + "zip_trace_reader.h", + ] + deps = [ + "../../../../gn:default_deps", + "../../../../include/perfetto/ext/base:base", + "../../../trace_processor:storage_minimal", + "../../types", + "../../util:trace_type", + "../../util:util", + "../../util:zip_reader", + "../android_bugreport", + "../common", + "../proto:minimal", + ] +} diff --git a/src/trace_processor/importers/zip/zip_trace_reader.cc b/src/trace_processor/importers/zip/zip_trace_reader.cc new file mode 100644 index 000000000..20f019f55 --- /dev/null +++ b/src/trace_processor/importers/zip/zip_trace_reader.cc @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/importers/zip/zip_trace_reader.h" + +#include <algorithm> +#include <cinttypes> +#include <cstdint> +#include <cstring> +#include <memory> +#include <string> +#include <tuple> +#include <utility> +#include <vector> + +#include "perfetto/base/logging.h" +#include "perfetto/base/status.h" +#include "perfetto/ext/base/status_or.h" +#include "perfetto/trace_processor/trace_blob.h" +#include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/forwarding_trace_parser.h" +#include "src/trace_processor/importers/android_bugreport/android_bugreport_parser.h" +#include "src/trace_processor/importers/proto/proto_trace_tokenizer.h" +#include "src/trace_processor/types/trace_processor_context.h" +#include "src/trace_processor/util/status_macros.h" +#include "src/trace_processor/util/trace_type.h" + +namespace perfetto { +namespace trace_processor { +namespace { + +// Proto traces should always parsed first as they might contains clock sync +// data needed to correctly parse other traces. +// The rest of the types are sorted by position in the enum but this is not +// something users should rely on. +// TODO(carlscab): Proto traces with just ModuleSymbols packets should be an +// exception. We actually need those are the very end (once whe have all the +// Frames). Alternatively we could build a map address -> symbol during +// tokenization and use this during parsing to resolve symbols. +bool CompareTraceType(TraceType lhs, TraceType rhs) { + if (rhs == TraceType::kProtoTraceType) { + return false; + } + if (lhs == TraceType::kProtoTraceType) { + return true; + } + return lhs < rhs; +} + +bool HasSymbols(const TraceBlobView& blob) { + bool has_symbols = false; + ProtoTraceTokenizer().Tokenize(blob.copy(), [&](TraceBlobView raw) { + protos::pbzero::TracePacket::Decoder packet(raw.data(), raw.size()); + has_symbols = packet.has_module_symbols(); + return base::ErrStatus("break"); + }); + return has_symbols; +} + +} // namespace + +ZipTraceReader::ZipTraceReader(TraceProcessorContext* context) + : context_(context) {} +ZipTraceReader::~ZipTraceReader() = default; + +bool ZipTraceReader::Entry::operator<(const Entry& rhs) const { + // Traces with symbols should be the last ones to be read. + if (has_symbols) { + return false; + } + if (rhs.has_symbols) { + return true; + } + if (CompareTraceType(trace_type, rhs.trace_type)) { + return true; + } + if (CompareTraceType(rhs.trace_type, trace_type)) { + return false; + } + return std::tie(name, index) < std::tie(rhs.name, rhs.index); +} + +util::Status ZipTraceReader::Parse(TraceBlobView blob) { + zip_reader_.Parse(blob.data(), blob.size()); + return base::OkStatus(); +} + +void ZipTraceReader::NotifyEndOfFile() { + base::Status status = NotifyEndOfFileImpl(); + if (!status.ok()) { + PERFETTO_ELOG("ZipTraceReader failed: %s", status.c_message()); + } +} + +base::Status ZipTraceReader::NotifyEndOfFileImpl() { + std::vector<util::ZipFile> files = zip_reader_.TakeFiles(); + + // Android bug reports are ZIP files and its files do not get handled + // separately. + if (AndroidBugreportParser::IsAndroidBugReport(files)) { + return AndroidBugreportParser::Parse(context_, std::move(files)); + } + + base::StatusOr<std::vector<Entry>> entries = ExtractEntries(std::move(files)); + if (!entries.ok()) { + return entries.status(); + } + std::sort(entries->begin(), entries->end()); + + for (Entry& e : *entries) { + parsers_.push_back(std::make_unique<ForwardingTraceParser>(context_)); + auto& parser = *parsers_.back(); + RETURN_IF_ERROR(parser.Parse(std::move(e.uncompressed_data))); + parser.NotifyEndOfFile(); + } + return base::OkStatus(); +} + +base::StatusOr<std::vector<ZipTraceReader::Entry>> +ZipTraceReader::ExtractEntries(std::vector<util::ZipFile> files) const { + // TODO(carlsacab): There is a lot of unnecessary copying going on here. + // ZipTraceReader can directly parse the ZIP file and given that we know the + // decompressed size we could directly decompress into TraceBlob chunks and + // send them to the tokenizer. + std::vector<Entry> entries; + std::vector<uint8_t> buffer; + for (size_t i = 0; i < files.size(); ++i) { + const util::ZipFile& zip_file = files[i]; + Entry entry; + entry.name = zip_file.name(); + entry.index = i; + RETURN_IF_ERROR(files[i].Decompress(&buffer)); + entry.uncompressed_data = + TraceBlobView(TraceBlob::CopyFrom(buffer.data(), buffer.size())); + entry.trace_type = GuessTraceType(entry.uncompressed_data.data(), + entry.uncompressed_data.size()); + entry.has_symbols = entry.trace_type == TraceType::kProtoTraceType && + HasSymbols(entry.uncompressed_data); + entries.push_back(std::move(entry)); + } + return std::move(entries); +} + +} // namespace trace_processor +} // namespace perfetto diff --git a/src/trace_processor/importers/zip/zip_trace_reader.h b/src/trace_processor/importers/zip/zip_trace_reader.h new file mode 100644 index 000000000..d4f3d8818 --- /dev/null +++ b/src/trace_processor/importers/zip/zip_trace_reader.h @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_ZIP_ZIP_TRACE_READER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_ZIP_ZIP_TRACE_READER_H_ + +#include <cstddef> +#include <memory> +#include <string> +#include <vector> + +#include "perfetto/base/status.h" +#include "perfetto/ext/base/status_or.h" +#include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/importers/common/chunked_trace_reader.h" +#include "src/trace_processor/util/trace_type.h" +#include "src/trace_processor/util/zip_reader.h" + +namespace perfetto::trace_processor { + +class ForwardingTraceParser; +class TraceProcessorContext; + +// Forwards files contained in a ZIP to the appropiate ChunkedTraceReader. It is +// guaranteed that proto traces will be parsed first. +class ZipTraceReader : public ChunkedTraceReader { + public: + explicit ZipTraceReader(TraceProcessorContext* context); + ~ZipTraceReader() override; + + // ChunkedTraceReader implementation + util::Status Parse(TraceBlobView) override; + void NotifyEndOfFile() override; + + private: + // Represents a file in the ZIP file. Used to sort them before sending the + // files one by one to a `ForwardingTraceParser` instance. + struct Entry { + // File name. Used to break ties. + std::string name; + // Position in the zip file. Used to break ties. + size_t index; + // Trace type. This is the main attribute traces are ordered by. Proto + // traces are always parsed first as they might contains clock sync + // data needed to correctly parse other traces. + TraceType trace_type; + TraceBlobView uncompressed_data; + // True for proto trace_types whose fist message is a ModuleSymbols packet + bool has_symbols = false; + // Comparator used to determine the order in which files in the ZIP will be + // read. + bool operator<(const Entry& rhs) const; + }; + + base::Status NotifyEndOfFileImpl(); + base::StatusOr<std::vector<Entry>> ExtractEntries( + std::vector<util::ZipFile> files) const; + base::Status ParseEntry(Entry entry); + + TraceProcessorContext* const context_; + util::ZipReader zip_reader_; + // For every file in the ZIP we will create a `ForwardingTraceParser`instance + // and send that file to it for tokenization. The instances are kept around + // here as some tokenizers might keep state that is later needed after + // sorting. + std::vector<std::unique_ptr<ForwardingTraceParser>> parsers_; +}; + +} // namespace perfetto::trace_processor + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_ZIP_ZIP_TRACE_READER_H_ |