simpleperf: dump file feature section.

For `record --dump-symbols` option, change from dumping
DsoRecord and SymbolRecord to dumping file feature section.
It is to avoid reading symbols from elf files during recording,
which takes a lot of time. And we don't want to mix optional
data (the symbol tables) with necessary data (the profiling records).

Bug: http://b/32340274
Test: run simpleperf_unit_test.
Test: run simpleperf runtest.py.

Change-Id: I0a387de243afac93486fc885f223a58060ec07f4

16 files changed, 366 insertions, 207 deletions

diff --git a/simpleperf/cmd_dumprecord.cpp b/simpleperf/cmd_dumprecord.cpp
index 047bfa23..191d641c 100644
--- a/simpleperf/cmd_dumprecord.cpp
+++ b/simpleperf/cmd_dumprecord.cpp
@@ -148,6 +148,7 @@ static const std::string GetFeatureName(int feature) {
       {FEAT_BRANCH_STACK, "branch_stack"},
       {FEAT_PMU_MAPPINGS, "pmu_mappings"},
       {FEAT_GROUP_DESC, "group_desc"},
+      {FEAT_FILE, "file"},
   };
   auto it = feature_name_map.find(feature);
   if (it != feature_name_map.end()) {
@@ -200,6 +201,25 @@ void DumpRecordCommand::DumpFeatureSection() {
     } else if (feature == FEAT_CMDLINE) {
       std::vector<std::string> cmdline = record_file_reader_->ReadCmdlineFeature();
       PrintIndented(1, "cmdline: %s\n", android::base::Join(cmdline, ' ').c_str());
+    } else if (feature == FEAT_FILE) {
+      std::string file_path;
+      uint32_t file_type;
+      uint64_t min_vaddr;
+      std::vector<Symbol> symbols;
+      size_t read_pos = 0;
+      PrintIndented(1, "file:\n");
+      while (record_file_reader_->ReadFileFeature(read_pos, &file_path,
+                                                  &file_type, &min_vaddr,
+                                                  &symbols)) {
+        PrintIndented(2, "file_path %s\n", file_path.c_str());
+        PrintIndented(2, "file_type %s\n", DsoTypeToString(static_cast<DsoType>(file_type)));
+        PrintIndented(2, "min_vaddr 0x%" PRIx64 "\n", min_vaddr);
+        PrintIndented(2, "symbols:\n");
+        for (const auto& symbol : symbols) {
+          PrintIndented(3, "%s [0x%" PRIx64 "-0x%" PRIx64 "]\n", symbol.DemangledName(),
+                        symbol.addr, symbol.addr + symbol.len);
+        }
+      }
     }
   }
 }
diff --git a/simpleperf/cmd_record.cpp b/simpleperf/cmd_record.cpp
index e3390b05..58f48132 100644
--- a/simpleperf/cmd_record.cpp
+++ b/simpleperf/cmd_record.cpp
@@ -182,12 +182,12 @@ class RecordCommand : public Command {
   bool DumpKernelAndModuleMmaps(const perf_event_attr& attr, uint64_t event_id);
   bool DumpThreadCommAndMmaps(const perf_event_attr& attr, uint64_t event_id);
   bool ProcessRecord(Record* record);
-  bool DumpSymbolForRecord(const SampleRecord& r, bool for_callchain);
   void UpdateRecordForEmbeddedElfPath(Record* record);
   bool UnwindRecord(Record* record);
   bool PostUnwind(const std::vector<std::string>& args);
   bool DumpAdditionalFeatures(const std::vector<std::string>& args);
   bool DumpBuildIdFeature();
+  bool DumpFileFeature();
   void CollectHitFileInfo(const SampleRecord& r);
 
   bool use_sample_freq_;
@@ -780,16 +780,6 @@ bool RecordCommand::ProcessRecord(Record* record) {
     sample_record_count_++;
     auto& r = *static_cast<SampleRecord*>(record);
     CollectHitFileInfo(r);
-    if (dump_symbols_) {
-      if (!DumpSymbolForRecord(r, false)) {
-        return false;
-      }
-      if (fp_callchain_sampling_) {
-        if (!DumpSymbolForRecord(r, true)) {
-          return false;
-        }
-      }
-    }
   } else if (record->type() == PERF_RECORD_LOST) {
     lost_record_count_ += static_cast<LostRecord*>(record)->lost;
   }
@@ -797,42 +787,6 @@ bool RecordCommand::ProcessRecord(Record* record) {
   return result;
 }
 
-bool RecordCommand::DumpSymbolForRecord(const SampleRecord& r,
-                                        bool for_callchain) {
-  const ThreadEntry* thread =
-      thread_tree_.FindThreadOrNew(r.tid_data.pid, r.tid_data.tid);
-  uint64_t ip_nr = 1;
-  const uint64_t* ips = &r.ip_data.ip;
-  if (for_callchain) {
-    ip_nr = r.callchain_data.ip_nr;
-    ips = r.callchain_data.ips;
-  }
-  for (uint64_t i = 0; i < ip_nr; ++i) {
-    const MapEntry* map = thread_tree_.FindMap(thread, ips[i], r.InKernel());
-    const Symbol* symbol = thread_tree_.FindSymbol(map, ips[i], nullptr);
-    if (symbol == thread_tree_.UnknownSymbol()) {
-      continue;
-    }
-    if (!map->dso->HasDumped()) {
-      map->dso->SetDumped();
-      DsoRecord dso_record(map->dso->type(), map->dso->id(), map->dso->Path(),
-                           map->dso->MinVirtualAddress());
-      if (!record_file_writer_->WriteRecord(dso_record)) {
-        return false;
-      }
-    }
-    if (!symbol->HasDumped()) {
-      symbol->SetDumped();
-      SymbolRecord symbol_record(symbol->addr, symbol->len, symbol->Name(),
-                                 map->dso->id());
-      if (!record_file_writer_->WriteRecord(symbol_record)) {
-        return false;
-      }
-    }
-  }
-  return true;
-}
-
 template <class RecordType>
 void UpdateMmapRecordForEmbeddedElfPath(RecordType* record) {
   RecordType& r = *record;
@@ -886,11 +840,6 @@ bool RecordCommand::UnwindRecord(Record* record) {
           UnwindCallChain(arch, *thread, regs, r.stack_user_data.data,
                           r.GetValidStackSize(), strict_arch_check);
       r.ReplaceRegAndStackWithCallChain(unwind_ips);
-      if (dump_symbols_) {
-        if (!DumpSymbolForRecord(r, true)) {
-          return false;
-        }
-      }
     }
   }
   return true;
@@ -941,13 +890,22 @@ bool RecordCommand::PostUnwind(const std::vector<std::string>& args) {
 
 bool RecordCommand::DumpAdditionalFeatures(
     const std::vector<std::string>& args) {
-  size_t feature_count = (branch_sampling_ != 0 ? 5 : 4);
-  if (!record_file_writer_->WriteFeatureHeader(feature_count)) {
+  size_t feature_count = 4;
+  if (branch_sampling_) {
+    feature_count++;
+  }
+  if (dump_symbols_) {
+    feature_count++;
+  }
+  if (!record_file_writer_->BeginWriteFeatures(feature_count)) {
     return false;
   }
   if (!DumpBuildIdFeature()) {
     return false;
   }
+  if (dump_symbols_ && !DumpFileFeature()) {
+    return false;
+  }
   utsname uname_buf;
   if (TEMP_FAILURE_RETRY(uname(&uname_buf)) != 0) {
     PLOG(ERROR) << "uname() failed";
@@ -975,6 +933,9 @@ bool RecordCommand::DumpAdditionalFeatures(
       !record_file_writer_->WriteBranchStackFeature()) {
     return false;
   }
+  if (!record_file_writer_->EndWriteFeatures()) {
+    return false;
+  }
   return true;
 }
 
@@ -1034,6 +995,23 @@ bool RecordCommand::DumpBuildIdFeature() {
   return true;
 }
 
+bool RecordCommand::DumpFileFeature() {
+  std::vector<Dso*> dso_v = thread_tree_.GetAllDsos();
+  for (Dso* dso : dso_v) {
+    if (!dso->IsHit()) {
+      continue;
+    }
+    uint32_t dso_type = dso->type();
+    uint64_t min_vaddr = dso->MinVirtualAddress();
+    const std::vector<Symbol>& symbols = dso->GetSymbols();
+    if (!record_file_writer_->WriteFileFeature(dso->Path(), dso_type, min_vaddr,
+                                               symbols)) {
+      return false;
+    }
+  }
+  return true;
+}
+
 void RecordCommand::CollectHitFileInfo(const SampleRecord& r) {
   const ThreadEntry* thread =
       thread_tree_.FindThreadOrNew(r.tid_data.pid, r.tid_data.tid);
diff --git a/simpleperf/cmd_report.cpp b/simpleperf/cmd_report.cpp
index 7fd0f61f..6a043c5f 100644
--- a/simpleperf/cmd_report.cpp
+++ b/simpleperf/cmd_report.cpp
@@ -629,6 +629,8 @@ bool ReportCommand::ReadEventAttrFromRecordFile() {
 }
 
 bool ReportCommand::ReadFeaturesFromRecordFile() {
+  // TODO: process features not only for report command, but also for
+  // other report ways: report_sample, report_lib_interface, etc.
   std::vector<BuildIdRecord> records =
       record_file_reader_->ReadBuildIdFeature();
   std::vector<std::pair<std::string, BuildId>> build_ids;
@@ -637,6 +639,18 @@ bool ReportCommand::ReadFeaturesFromRecordFile() {
   }
   Dso::SetBuildIds(build_ids);
 
+  if (record_file_reader_->HasFeature(PerfFileFormat::FEAT_FILE)) {
+    std::string file_path;
+    uint32_t file_type;
+    uint64_t min_vaddr;
+    std::vector<Symbol> symbols;
+    size_t read_pos = 0;
+    while (record_file_reader_->ReadFileFeature(
+        read_pos, &file_path, &file_type, &min_vaddr, &symbols)) {
+      thread_tree_.AddDsoInfo(file_path, file_type, min_vaddr, &symbols);
+    }
+  }
+
   std::string arch =
       record_file_reader_->ReadFeatureString(PerfFileFormat::FEAT_ARCH);
   if (!arch.empty()) {
diff --git a/simpleperf/cmd_report_test.cpp b/simpleperf/cmd_report_test.cpp
index e00b5ee6..98190ee1 100644
--- a/simpleperf/cmd_report_test.cpp
+++ b/simpleperf/cmd_report_test.cpp
@@ -340,7 +340,7 @@ TEST_F(ReportCommandTest, report_dumped_symbols) {
   ASSERT_NE(content.find("main"), std::string::npos);
   Report(PERF_DATA_WITH_SYMBOLS_FOR_NONZERO_MINVADDR_DSO);
   ASSERT_TRUE(success);
-  ASSERT_NE(content.find("main"), std::string::npos);
+  ASSERT_NE(content.find("memcpy"), std::string::npos);
 }
 
 TEST_F(ReportCommandTest, report_sort_vaddr_in_file) {
diff --git a/simpleperf/dso.cpp b/simpleperf/dso.cpp
index b22ebbf2..2950939c 100644
--- a/simpleperf/dso.cpp
+++ b/simpleperf/dso.cpp
@@ -127,18 +127,15 @@ BuildId Dso::GetExpectedBuildId() {
 
 std::unique_ptr<Dso> Dso::CreateDso(DsoType dso_type,
                                     const std::string& dso_path) {
-  static uint64_t id = 0;
-  return std::unique_ptr<Dso>(new Dso(dso_type, ++id, dso_path));
+  return std::unique_ptr<Dso>(new Dso(dso_type, dso_path));
 }
 
-Dso::Dso(DsoType type, uint64_t id, const std::string& path)
+Dso::Dso(DsoType type, const std::string& path)
     : type_(type),
-      id_(id),
       path_(path),
       debug_file_path_(path),
       min_vaddr_(std::numeric_limits<uint64_t>::max()),
       is_loaded_(false),
-      has_dumped_(false),
       hit_flag_(false) {
   // Check if file matching path_ exists in symfs directory before using it as
   // debug_file_path_.
@@ -173,35 +170,48 @@ Dso::~Dso() {
   }
 }
 
+static bool CompareSymbol(const Symbol& symbol1, const Symbol& symbol2) {
+  return symbol1.addr < symbol2.addr;
+}
+
 const Symbol* Dso::FindSymbol(uint64_t vaddr_in_dso) {
   if (!is_loaded_) {
-    is_loaded_ = true;
-    // If symbols has been read from SymbolRecords, no need to load them from
-    // dso.
-    if (symbols_.empty()) {
-      if (!Load()) {
-        LOG(DEBUG) << "failed to load dso: " << path_;
-        return nullptr;
+    Load();
+  }
+  if (!symbols_.empty()) {
+    auto it = std::upper_bound(symbols_.begin(), symbols_.end(), Symbol("", vaddr_in_dso, 0), CompareSymbol);
+    if (it != symbols_.begin()) {
+      --it;
+      if (it->addr <= vaddr_in_dso && (it->addr + it->len > vaddr_in_dso)) {
+        return &*it;
       }
     }
   }
-  if (symbols_.empty()) {
-    return nullptr;
-  }
-
-  auto it = symbols_.upper_bound(Symbol("", vaddr_in_dso, 0));
-  if (it != symbols_.begin()) {
-    --it;
-    // If vaddr_in_dso is ULLONG_MAX, then it->addr + it->len overflows,
-    // and we allow this situation.
-    if (it->addr <= vaddr_in_dso && (it->addr + it->len > vaddr_in_dso ||
-                                     it->addr + it->len < it->addr)) {
-      return &*it;
+  if (!unknown_symbols_.empty()) {
+    auto it = unknown_symbols_.find(vaddr_in_dso);
+    if (it != unknown_symbols_.end()) {
+      return &it->second;
     }
   }
   return nullptr;
 }
 
+const std::vector<Symbol>& Dso::GetSymbols() {
+  if (!is_loaded_) {
+    Load();
+  }
+  return symbols_;
+}
+
+void Dso::SetSymbols(std::vector<Symbol>* symbols) {
+  symbols_ = std::move(*symbols);
+  symbols->clear();
+}
+
+void Dso::AddUnknownSymbol(uint64_t vaddr_in_dso, const std::string& name) {
+  unknown_symbols_.insert(std::make_pair(vaddr_in_dso, Symbol(name, vaddr_in_dso, 1)));
+}
+
 uint64_t Dso::MinVirtualAddress() {
   if (min_vaddr_ == std::numeric_limits<uint64_t>::max()) {
     min_vaddr_ = 0;
@@ -222,7 +232,14 @@ uint64_t Dso::MinVirtualAddress() {
   return min_vaddr_;
 }
 
-bool Dso::Load() {
+void Dso::Load() {
+  is_loaded_ = true;
+  if (!symbols_.empty()) {
+    // If symbols has been read from file feature section of perf.data, no
+    // need to load them from file system.
+    // TODO: combine symbols in file feature section and in file system.
+    return;
+  }
   bool result = false;
   switch (type_) {
     case DSO_KERNEL:
@@ -241,11 +258,12 @@ bool Dso::Load() {
     }
   }
   if (result) {
+    std::sort(symbols_.begin(), symbols_.end(), CompareSymbol);
     FixupSymbolLength();
   } else {
     symbols_.clear();
+    LOG(DEBUG) << "failed to load dso: " << path_;
   }
-  return result;
 }
 
 static bool IsKernelFunctionSymbol(const KernelSymbol& symbol) {
@@ -253,16 +271,18 @@ static bool IsKernelFunctionSymbol(const KernelSymbol& symbol) {
           symbol.type == 'w');
 }
 
-static bool KernelSymbolCallback(const KernelSymbol& kernel_symbol, Dso* dso) {
+static bool KernelSymbolCallback(const KernelSymbol& kernel_symbol,
+                                 std::vector<Symbol>* symbols) {
   if (IsKernelFunctionSymbol(kernel_symbol)) {
-    dso->InsertSymbol(Symbol(kernel_symbol.name, kernel_symbol.addr, 0));
+    symbols->emplace_back(Symbol(kernel_symbol.name, kernel_symbol.addr, 0));
   }
   return false;
 }
 
-static void VmlinuxSymbolCallback(const ElfFileSymbol& elf_symbol, Dso* dso) {
+static void VmlinuxSymbolCallback(const ElfFileSymbol& elf_symbol,
+                                  std::vector<Symbol>* symbols) {
   if (elf_symbol.is_func) {
-    dso->InsertSymbol(
+    symbols->emplace_back(
         Symbol(elf_symbol.name, elf_symbol.vaddr, elf_symbol.len));
   }
 }
@@ -286,11 +306,11 @@ bool Dso::LoadKernel() {
   BuildId build_id = GetExpectedBuildId();
   if (!vmlinux_.empty()) {
     ElfStatus result = ParseSymbolsFromElfFile(vmlinux_, build_id,
-        std::bind(VmlinuxSymbolCallback, std::placeholders::_1, this));
+        std::bind(VmlinuxSymbolCallback, std::placeholders::_1, &symbols_));
     return CheckReadSymbolResult(result, vmlinux_);
   } else if (!kallsyms_.empty()) {
     ProcessKernelSymbols(kallsyms_, std::bind(&KernelSymbolCallback,
-                                              std::placeholders::_1, this));
+                                              std::placeholders::_1, &symbols_));
     bool all_zero = true;
     for (const auto& symbol : symbols_) {
       if (symbol.addr != 0) {
@@ -325,7 +345,7 @@ bool Dso::LoadKernel() {
       return false;
     }
     ProcessKernelSymbols(kallsyms, std::bind(&KernelSymbolCallback,
-                                             std::placeholders::_1, this));
+                                             std::placeholders::_1, &symbols_));
     bool all_zero = true;
     for (const auto& symbol : symbols_) {
       if (symbol.addr != 0) {
@@ -343,11 +363,11 @@ bool Dso::LoadKernel() {
   return true;
 }
 
-static void ElfFileSymbolCallback(const ElfFileSymbol& elf_symbol, Dso* dso,
-                                  bool (*filter)(const ElfFileSymbol&)) {
+static void ElfFileSymbolCallback(const ElfFileSymbol& elf_symbol,
+                                  bool (*filter)(const ElfFileSymbol&),
+                                  std::vector<Symbol>* symbols) {
   if (filter(elf_symbol)) {
-    dso->InsertSymbol(
-        Symbol(elf_symbol.name, elf_symbol.vaddr, elf_symbol.len));
+    symbols->emplace_back(elf_symbol.name, elf_symbol.vaddr, elf_symbol.len);
   }
 }
 
@@ -359,8 +379,8 @@ static bool SymbolFilterForKernelModule(const ElfFileSymbol& elf_symbol) {
 bool Dso::LoadKernelModule() {
   BuildId build_id = GetExpectedBuildId();
   ElfStatus result = ParseSymbolsFromElfFile(GetDebugFilePath(), build_id,
-      std::bind(ElfFileSymbolCallback, std::placeholders::_1, this,
-                SymbolFilterForKernelModule));
+      std::bind(ElfFileSymbolCallback, std::placeholders::_1,
+                SymbolFilterForKernelModule, &symbols_));
   return CheckReadSymbolResult(result, GetDebugFilePath());
 }
 
@@ -376,16 +396,18 @@ bool Dso::LoadElfFile() {
     // Linux host can store debug shared libraries in /usr/lib/debug.
     ElfStatus result = ParseSymbolsFromElfFile(
         "/usr/lib/debug" + path_, build_id,
-        std::bind(ElfFileSymbolCallback, std::placeholders::_1, this,
-                  SymbolFilterForDso));
+        std::bind(ElfFileSymbolCallback, std::placeholders::_1,
+                  SymbolFilterForDso, &symbols_));
     if (result == ElfStatus::NO_ERROR) {
       return CheckReadSymbolResult(result, "/usr/lib/debug" + path_);
     }
   }
+  // TODO: load std::vector<Symbol> directly from ParseSymbolsFromElfFile
+  // instead of needing to call a callback function for each symbol.
   ElfStatus result = ParseSymbolsFromElfFile(
       GetDebugFilePath(), build_id,
-      std::bind(ElfFileSymbolCallback, std::placeholders::_1, this,
-                SymbolFilterForDso));
+      std::bind(ElfFileSymbolCallback, std::placeholders::_1,
+                SymbolFilterForDso, &symbols_));
   return CheckReadSymbolResult(result, GetDebugFilePath());
 }
 
@@ -395,13 +417,11 @@ bool Dso::LoadEmbeddedElfFile() {
   CHECK(std::get<0>(tuple));
   ElfStatus result = ParseSymbolsFromApkFile(
       std::get<1>(tuple), std::get<2>(tuple), build_id,
-      std::bind(ElfFileSymbolCallback, std::placeholders::_1, this,
-                SymbolFilterForDso));
+      std::bind(ElfFileSymbolCallback, std::placeholders::_1,
+                SymbolFilterForDso, &symbols_));
   return CheckReadSymbolResult(result, GetDebugFilePath());
 }
 
-void Dso::InsertSymbol(const Symbol& symbol) { symbols_.insert(symbol); }
-
 void Dso::FixupSymbolLength() {
   Symbol* prev_symbol = nullptr;
   for (auto& symbol : symbols_) {
diff --git a/simpleperf/dso.h b/simpleperf/dso.h
index 1d53f3ff..c464b733 100644
--- a/simpleperf/dso.h
+++ b/simpleperf/dso.h
@@ -18,7 +18,6 @@
 #define SIMPLE_PERF_DSO_H_
 
 #include <memory>
-#include <set>
 #include <string>
 #include <unordered_map>
 #include <vector>
@@ -44,12 +43,6 @@ struct Symbol {
   mutable bool has_dumped_;
 };
 
-struct SymbolComparator {
-  bool operator()(const Symbol& symbol1, const Symbol& symbol2) {
-    return symbol1.addr < symbol2.addr;
-  }
-};
-
 enum DsoType {
   DSO_KERNEL,
   DSO_KERNEL_MODULE,
@@ -80,8 +73,6 @@ struct Dso {
 
   DsoType type() const { return type_; }
 
-  uint64_t id() const { return id_; }
-
   // Return the path recorded in perf.data.
   const std::string& Path() const { return path_; }
   // Return the path containing symbol table and debug information.
@@ -89,10 +80,6 @@ struct Dso {
   // Return the file name without directory info.
   const std::string& FileName() const { return file_name_; }
 
-  bool HasDumped() const { return has_dumped_; }
-
-  void SetDumped() { has_dumped_ = true; }
-
   // Set when there are samples hit in current dso.
   void SetHitFlag() { hit_flag_ = true; }
   bool IsHit() const { return hit_flag_; }
@@ -102,7 +89,13 @@ struct Dso {
   void SetMinVirtualAddress(uint64_t min_vaddr) { min_vaddr_ = min_vaddr; }
 
   const Symbol* FindSymbol(uint64_t vaddr_in_dso);
-  void InsertSymbol(const Symbol& symbol);
+
+  const std::vector<Symbol>& GetSymbols();
+  void SetSymbols(std::vector<Symbol>* symbols);
+
+  // Create a symbol for a virtual address which can't find a corresponding
+  // symbol in symbol table.
+  void AddUnknownSymbol(uint64_t vaddr_in_dso, const std::string& name);
 
  private:
   static bool demangle_;
@@ -112,8 +105,8 @@ struct Dso {
   static std::unordered_map<std::string, BuildId> build_id_map_;
   static size_t dso_count_;
 
-  Dso(DsoType type, uint64_t id, const std::string& path);
-  bool Load();
+  Dso(DsoType type, const std::string& path);
+  void Load();
   bool LoadKernel();
   bool LoadKernelModule();
   bool LoadElfFile();
@@ -122,7 +115,6 @@ struct Dso {
   BuildId GetExpectedBuildId();
 
   const DsoType type_;
-  const uint64_t id_;
   // path of the shared library used by the profiled program
   const std::string path_;
   // path of the shared library having symbol table and debug information
@@ -131,9 +123,10 @@ struct Dso {
   // File name of the shared library, got by removing directories in path_.
   std::string file_name_;
   uint64_t min_vaddr_;
-  std::set<Symbol, SymbolComparator> symbols_;
+  std::vector<Symbol> symbols_;
+  // unknown symbols are like [libc.so+0x1234].
+  std::unordered_map<uint64_t, Symbol> unknown_symbols_;
   bool is_loaded_;
-  bool has_dumped_;
   bool hit_flag_;
 };
 
diff --git a/simpleperf/record.h b/simpleperf/record.h
index 22c988ad..4be46d9f 100644
--- a/simpleperf/record.h
+++ b/simpleperf/record.h
@@ -40,6 +40,7 @@ enum user_record_type {
 
   SIMPLE_PERF_RECORD_TYPE_START = 32768,
   SIMPLE_PERF_RECORD_KERNEL_SYMBOL,
+  // TODO: remove DsoRecord and SymbolRecord.
   SIMPLE_PERF_RECORD_DSO,
   SIMPLE_PERF_RECORD_SYMBOL,
   SIMPLE_PERF_RECORD_SPLIT,
diff --git a/simpleperf/record_file.h b/simpleperf/record_file.h
index 9594d166..4d93c158 100644
--- a/simpleperf/record_file.h
+++ b/simpleperf/record_file.h
@@ -28,6 +28,7 @@
 
 #include <android-base/macros.h>
 
+#include "dso.h"
 #include "event_attr.h"
 #include "perf_event.h"
 #include "record.h"
@@ -43,11 +44,16 @@ class RecordFileWriter {
   bool WriteAttrSection(const std::vector<EventAttrWithId>& attr_ids);
   bool WriteRecord(const Record& record);
 
-  bool WriteFeatureHeader(size_t feature_count);
+  bool BeginWriteFeatures(size_t feature_count);
   bool WriteBuildIdFeature(const std::vector<BuildIdRecord>& build_id_records);
   bool WriteFeatureString(int feature, const std::string& s);
   bool WriteCmdlineFeature(const std::vector<std::string>& cmdline);
   bool WriteBranchStackFeature();
+  bool WriteFileFeature(const std::string& file_path,
+                        uint32_t file_type,
+                        uint64_t min_vaddr,
+                        const std::vector<Symbol>& symbols);
+  bool EndWriteFeatures();
 
   // Normally, Close() should be called after writing. But if something
   // wrong happens and we need to finish in advance, the destructor
@@ -62,9 +68,10 @@ class RecordFileWriter {
   bool WriteFileHeader();
   bool WriteData(const void* buf, size_t len);
   bool Write(const void* buf, size_t len);
-  bool SeekFileEnd(uint64_t* file_end);
-  bool WriteFeatureBegin(uint64_t* start_offset);
-  bool WriteFeatureEnd(int feature, uint64_t start_offset);
+  bool GetFilePos(uint64_t* file_pos);
+  bool WriteStringWithLength(const std::string& s);
+  bool WriteFeatureBegin(int feature);
+  bool WriteFeatureEnd(int feature);
 
   const std::string filename_;
   FILE* record_fp_;
@@ -74,10 +81,10 @@ class RecordFileWriter {
   uint64_t attr_section_size_;
   uint64_t data_section_offset_;
   uint64_t data_section_size_;
+  uint64_t feature_section_offset_;
 
-  std::vector<int> features_;
-  int feature_count_;
-  int current_feature_index_;
+  std::map<int, PerfFileFormat::SectionDesc> features_;
+  size_t feature_count_;
 
   DISALLOW_COPY_AND_ASSIGN(RecordFileWriter);
 };
@@ -128,6 +135,15 @@ class RecordFileReader {
   std::vector<std::string> ReadCmdlineFeature();
   std::vector<BuildIdRecord> ReadBuildIdFeature();
   std::string ReadFeatureString(int feature);
+
+  // File feature section contains many file information. This function reads
+  // one file information located at [read_pos]. [read_pos] is 0 at the first
+  // call, and is updated to point to the next file information. Return true
+  // if read successfully, and return false if there is no more file
+  // information.
+  bool ReadFileFeature(size_t& read_pos, std::string* file_path,
+                       uint32_t* file_type, uint64_t* min_vaddr,
+                       std::vector<Symbol>* symbols);
   bool Close();
 
   // For testing only.
diff --git a/simpleperf/record_file_format.h b/simpleperf/record_file_format.h
index da6434b0..efa000cf 100644
--- a/simpleperf/record_file_format.h
+++ b/simpleperf/record_file_format.h
@@ -32,6 +32,22 @@
 //    data section of feature 2
 //    ....
 
+// file feature section:
+//  file_struct files[];
+//
+//  struct file_struct {
+//    uint32_t size;  // size of rest fields in file_struct
+//    char file_path[];
+//    uint32_t file_type;
+//    uint64_t min_vaddr;
+//    uint32_t symbol_count;
+//    struct {
+//      uint64_t start_vaddr;
+//      uint32_t len;
+//      char symbol_name[];
+//    } symbol_table;
+//  };
+
 namespace PerfFileFormat {
 
 enum {
@@ -55,6 +71,9 @@ enum {
   FEAT_PMU_MAPPINGS,
   FEAT_GROUP_DESC,
   FEAT_LAST_FEATURE,
+
+  FEAT_SIMPLEPERF_START = 128,
+  FEAT_FILE = FEAT_SIMPLEPERF_START,
   FEAT_MAX_NUM = 256,
 };
 
diff --git a/simpleperf/record_file_reader.cpp b/simpleperf/record_file_reader.cpp
index c300fc72..b58b0ce7 100644
--- a/simpleperf/record_file_reader.cpp
+++ b/simpleperf/record_file_reader.cpp
@@ -378,6 +378,60 @@ std::string RecordFileReader::ReadFeatureString(int feature) {
   return p;
 }
 
+bool RecordFileReader::ReadFileFeature(size_t& read_pos,
+                                       std::string* file_path,
+                                       uint32_t* file_type,
+                                       uint64_t* min_vaddr,
+                                       std::vector<Symbol>* symbols) {
+  auto it = feature_section_descriptors_.find(FEAT_FILE);
+  if (it == feature_section_descriptors_.end()) {
+    return false;
+  }
+  if (read_pos >= it->second.size) {
+    return false;
+  }
+  if (read_pos == 0) {
+    if (fseek(record_fp_, it->second.offset, SEEK_SET) != 0) {
+      PLOG(ERROR) << "fseek() failed";
+      return false;
+    }
+  }
+  uint32_t size;
+  if (!Read(&size, 4)) {
+    return false;
+  }
+  std::vector<char> buf(size);
+  if (!Read(buf.data(), size)) {
+    return false;
+  }
+  read_pos += 4 + size;
+  const char* p = buf.data();
+  *file_path = p;
+  p += file_path->size() + 1;
+  memcpy(file_type, p, sizeof(uint32_t));
+  p += sizeof(uint32_t);
+  memcpy(min_vaddr, p, sizeof(uint64_t));
+  p += sizeof(uint64_t);
+  uint32_t symbol_count;
+  memcpy(&symbol_count, p, sizeof(uint32_t));
+  p += sizeof(uint32_t);
+  symbols->clear();
+  symbols->reserve(symbol_count);
+  for (uint32_t i = 0; i < symbol_count; ++i) {
+    uint64_t start_vaddr;
+    uint32_t len;
+    memcpy(&start_vaddr, p, sizeof(uint64_t));
+    p += sizeof(uint64_t);
+    memcpy(&len, p, sizeof(uint32_t));
+    p += sizeof(uint32_t);
+    std::string name = p;
+    p += name.size() + 1;
+    symbols->emplace_back(name, start_vaddr, len);
+  }
+  CHECK_EQ(size, static_cast<size_t>(p - buf.data()));
+  return true;
+}
+
 std::vector<std::unique_ptr<Record>> RecordFileReader::DataSection() {
   std::vector<std::unique_ptr<Record>> records;
   ReadDataSection([&](std::unique_ptr<Record> record) {
diff --git a/simpleperf/record_file_test.cpp b/simpleperf/record_file_test.cpp
index 2d619a2f..4fe725b9 100644
--- a/simpleperf/record_file_test.cpp
+++ b/simpleperf/record_file_test.cpp
@@ -66,7 +66,7 @@ TEST_F(RecordFileTest, smoke) {
   ASSERT_TRUE(writer->WriteRecord(mmap_record));
 
   // Write feature section.
-  ASSERT_TRUE(writer->WriteFeatureHeader(1));
+  ASSERT_TRUE(writer->BeginWriteFeatures(1));
   char p[BuildId::Size()];
   for (size_t i = 0; i < BuildId::Size(); ++i) {
     p[i] = i;
@@ -75,6 +75,7 @@ TEST_F(RecordFileTest, smoke) {
   std::vector<BuildIdRecord> build_id_records;
   build_id_records.push_back(BuildIdRecord(false, getpid(), build_id, "init"));
   ASSERT_TRUE(writer->WriteBuildIdFeature(build_id_records));
+  ASSERT_TRUE(writer->EndWriteFeatures());
   ASSERT_TRUE(writer->Close());
 
   // Read from a record file.
diff --git a/simpleperf/record_file_writer.cpp b/simpleperf/record_file_writer.cpp
index f90f5727..1eb072a3 100644
--- a/simpleperf/record_file_writer.cpp
+++ b/simpleperf/record_file_writer.cpp
@@ -58,8 +58,8 @@ RecordFileWriter::RecordFileWriter(const std::string& filename, FILE* fp)
       attr_section_size_(0),
       data_section_offset_(0),
       data_section_size_(0),
-      feature_count_(0),
-      current_feature_index_(0) {
+      feature_section_offset_(0),
+      feature_count_(0) {
 }
 
 RecordFileWriter::~RecordFileWriter() {
@@ -79,8 +79,8 @@ bool RecordFileWriter::WriteAttrSection(const std::vector<EventAttrWithId>& attr
   }
 
   // Write id section.
-  off_t id_section_offset = ftello(record_fp_);
-  if (id_section_offset == -1) {
+  uint64_t id_section_offset;
+  if (!GetFilePos(&id_section_offset)) {
     return false;
   }
   for (auto& attr_id : attr_ids) {
@@ -90,8 +90,8 @@ bool RecordFileWriter::WriteAttrSection(const std::vector<EventAttrWithId>& attr
   }
 
   // Write attr section.
-  off_t attr_section_offset = ftello(record_fp_);
-  if (attr_section_offset == -1) {
+  uint64_t attr_section_offset;
+  if (!GetFilePos(&attr_section_offset)) {
     return false;
   }
   for (auto& attr_id : attr_ids) {
@@ -105,8 +105,8 @@ bool RecordFileWriter::WriteAttrSection(const std::vector<EventAttrWithId>& attr
     }
   }
 
-  off_t data_section_offset = ftello(record_fp_);
-  if (data_section_offset == -1) {
+  uint64_t data_section_offset;
+  if (!GetFilePos(&data_section_offset)) {
     return false;
   }
 
@@ -173,28 +173,24 @@ bool RecordFileWriter::Write(const void* buf, size_t len) {
   return true;
 }
 
-bool RecordFileWriter::SeekFileEnd(uint64_t* file_end) {
-  if (fseek(record_fp_, 0, SEEK_END) == -1) {
-    PLOG(ERROR) << "fseek() failed";
-    return false;
-  }
+bool RecordFileWriter::GetFilePos(uint64_t* file_pos) {
   off_t offset = ftello(record_fp_);
   if (offset == -1) {
     PLOG(ERROR) << "ftello() failed";
     return false;
   }
-  *file_end = static_cast<uint64_t>(offset);
+  *file_pos = static_cast<uint64_t>(offset);
   return true;
 }
 
-bool RecordFileWriter::WriteFeatureHeader(size_t feature_count) {
+bool RecordFileWriter::BeginWriteFeatures(size_t feature_count) {
+  feature_section_offset_ = data_section_offset_ + data_section_size_;
   feature_count_ = feature_count;
-  current_feature_index_ = 0;
   uint64_t feature_header_size = feature_count * sizeof(SectionDesc);
 
   // Reserve enough space in the record file for the feature header.
   std::vector<unsigned char> zero_data(feature_header_size);
-  if (fseek(record_fp_, data_section_offset_ + data_section_size_, SEEK_SET) == -1) {
+  if (fseek(record_fp_, feature_section_offset_, SEEK_SET) == -1) {
     PLOG(ERROR) << "fseek() failed";
     return false;
   }
@@ -202,8 +198,7 @@ bool RecordFileWriter::WriteFeatureHeader(size_t feature_count) {
 }
 
 bool RecordFileWriter::WriteBuildIdFeature(const std::vector<BuildIdRecord>& build_id_records) {
-  uint64_t start_offset;
-  if (!WriteFeatureBegin(&start_offset)) {
+  if (!WriteFeatureBegin(FEAT_BUILD_ID)) {
     return false;
   }
   for (auto& record : build_id_records) {
@@ -211,29 +206,40 @@ bool RecordFileWriter::WriteBuildIdFeature(const std::vector<BuildIdRecord>& bui
       return false;
     }
   }
-  return WriteFeatureEnd(FEAT_BUILD_ID, start_offset);
+  return WriteFeatureEnd(FEAT_BUILD_ID);
 }
 
-bool RecordFileWriter::WriteFeatureString(int feature, const std::string& s) {
-  uint64_t start_offset;
-  if (!WriteFeatureBegin(&start_offset)) {
-    return false;
-  }
+bool RecordFileWriter::WriteStringWithLength(const std::string& s) {
   uint32_t len = static_cast<uint32_t>(Align(s.size() + 1, 64));
   if (!Write(&len, sizeof(len))) {
     return false;
   }
-  std::vector<char> v(len, '\0');
-  std::copy(s.begin(), s.end(), v.begin());
-  if (!Write(v.data(), v.size())) {
+  if (!Write(&s[0], s.size() + 1)) {
+    return false;
+  }
+  size_t pad_size = Align(s.size() + 1, 64) - s.size() - 1;
+  if (pad_size > 0u) {
+    char align_buf[pad_size];
+    memset(align_buf, '\0', pad_size);
+    if (!Write(align_buf, pad_size)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+bool RecordFileWriter::WriteFeatureString(int feature, const std::string& s) {
+  if (!WriteFeatureBegin(feature)) {
+    return false;
+  }
+  if (!WriteStringWithLength(s)) {
     return false;
   }
-  return WriteFeatureEnd(feature, start_offset);
+  return WriteFeatureEnd(feature);
 }
 
 bool RecordFileWriter::WriteCmdlineFeature(const std::vector<std::string>& cmdline) {
-  uint64_t start_offset;
-  if (!WriteFeatureBegin(&start_offset)) {
+  if (!WriteFeatureBegin(FEAT_CMDLINE)) {
     return false;
   }
   uint32_t arg_count = cmdline.size();
@@ -241,54 +247,91 @@ bool RecordFileWriter::WriteCmdlineFeature(const std::vector<std::string>& cmdli
     return false;
   }
   for (auto& arg : cmdline) {
-    uint32_t len = static_cast<uint32_t>(Align(arg.size() + 1, 64));
-    if (!Write(&len, sizeof(len))) {
-      return false;
-    }
-    std::vector<char> array(len, '\0');
-    std::copy(arg.begin(), arg.end(), array.begin());
-    if (!Write(array.data(), array.size())) {
+    if (!WriteStringWithLength(arg)) {
       return false;
     }
   }
-  return WriteFeatureEnd(FEAT_CMDLINE, start_offset);
+  return WriteFeatureEnd(FEAT_CMDLINE);
 }
 
 bool RecordFileWriter::WriteBranchStackFeature() {
-  uint64_t start_offset;
-  if (!WriteFeatureBegin(&start_offset)) {
+  if (!WriteFeatureBegin(FEAT_BRANCH_STACK)) {
     return false;
   }
-  return WriteFeatureEnd(FEAT_BRANCH_STACK, start_offset);
+  return WriteFeatureEnd(FEAT_BRANCH_STACK);
 }
 
-bool RecordFileWriter::WriteFeatureBegin(uint64_t* start_offset) {
-  CHECK_LT(current_feature_index_, feature_count_);
-  if (!SeekFileEnd(start_offset)) {
+bool RecordFileWriter::WriteFileFeature(const std::string& file_path,
+                                        uint32_t file_type,
+                                        uint64_t min_vaddr,
+                                        const std::vector<Symbol>& symbols) {
+  uint32_t size = file_path.size() + 1 + sizeof(uint32_t) * 2 +
+      sizeof(uint64_t) + symbols.size() * (sizeof(uint64_t) + sizeof(uint32_t));
+  for (const auto& symbol : symbols) {
+    size += strlen(symbol.Name()) + 1;
+  }
+  std::vector<char> buf(sizeof(uint32_t) + size);
+  char* p = buf.data();
+  MoveToBinaryFormat(size, p);
+  MoveToBinaryFormat(file_path.c_str(), file_path.size() + 1, p);
+  MoveToBinaryFormat(file_type, p);
+  MoveToBinaryFormat(min_vaddr, p);
+  uint32_t symbol_count = static_cast<uint32_t>(symbols.size());
+  MoveToBinaryFormat(symbol_count, p);
+  for (const auto& symbol : symbols) {
+    MoveToBinaryFormat(symbol.addr, p);
+    uint32_t len = symbol.len;
+    MoveToBinaryFormat(len, p);
+    MoveToBinaryFormat(symbol.Name(), strlen(symbol.Name()) + 1, p);
+  }
+  CHECK_EQ(buf.size(), static_cast<size_t>(p - buf.data()));
+
+  if (!WriteFeatureBegin(FEAT_FILE)) {
     return false;
   }
+  if (!Write(buf.data(), buf.size())) {
+    return false;
+  }
+  return WriteFeatureEnd(FEAT_FILE);
+}
+
+bool RecordFileWriter::WriteFeatureBegin(int feature) {
+  auto it = features_.find(feature);
+  if (it == features_.end()) {
+    CHECK_LT(features_.size(), feature_count_);
+    auto& sec = features_[feature];
+    if (!GetFilePos(&sec.offset)) {
+      return false;
+    }
+    sec.size = 0;
+  }
   return true;
 }
 
-bool RecordFileWriter::WriteFeatureEnd(int feature, uint64_t start_offset) {
-  uint64_t end_offset;
-  if (!SeekFileEnd(&end_offset)) {
+bool RecordFileWriter::WriteFeatureEnd(int feature) {
+  auto it = features_.find(feature);
+  if (it == features_.end()) {
     return false;
   }
-  SectionDesc desc;
-  desc.offset = start_offset;
-  desc.size = end_offset - start_offset;
-  uint64_t feature_offset = data_section_offset_ + data_section_size_;
-  if (fseek(record_fp_, feature_offset + current_feature_index_ * sizeof(SectionDesc), SEEK_SET) ==
-      -1) {
-    PLOG(ERROR) << "fseek() failed";
+  uint64_t offset;
+  if (!GetFilePos(&offset)) {
     return false;
   }
-  if (!Write(&desc, sizeof(SectionDesc))) {
+  it->second.size = offset - it->second.offset;
+  return true;
+}
+
+bool RecordFileWriter::EndWriteFeatures() {
+  CHECK_LE(feature_count_, features_.size());
+  if (fseek(record_fp_, feature_section_offset_, SEEK_SET) == -1) {
+    PLOG(ERROR) << "fseek() failed";
     return false;
   }
-  ++current_feature_index_;
-  features_.push_back(feature);
+  for (const auto& pair : features_) {
+    if (!Write(&pair.second, sizeof(SectionDesc))) {
+      return false;
+    }
+  }
   return true;
 }
 
@@ -302,9 +345,9 @@ bool RecordFileWriter::WriteFileHeader() {
   header.attrs.size = attr_section_size_;
   header.data.offset = data_section_offset_;
   header.data.size = data_section_size_;
-  for (auto& feature : features_) {
-    int i = feature / 8;
-    int j = feature % 8;
+  for (const auto& pair : features_) {
+    int i = pair.first / 8;
+    int j = pair.first % 8;
     header.features[i] |= (1 << j);
   }
 
diff --git a/simpleperf/testdata/perf_with_symbols.data b/simpleperf/testdata/perf_with_symbols.data
index 8571a964..ca74d159 100644
--- a/simpleperf/testdata/perf_with_symbols.data
+++ b/simpleperf/testdata/perf_with_symbols.data
diff --git a/simpleperf/testdata/perf_with_symbols_for_nonzero_minvaddr_dso.data b/simpleperf/testdata/perf_with_symbols_for_nonzero_minvaddr_dso.data
index cf45d3aa..b5fb92df 100644
--- a/simpleperf/testdata/perf_with_symbols_for_nonzero_minvaddr_dso.data
+++ b/simpleperf/testdata/perf_with_symbols_for_nonzero_minvaddr_dso.data
diff --git a/simpleperf/thread_tree.cpp b/simpleperf/thread_tree.cpp
index 69dab725..5868502b 100644
--- a/simpleperf/thread_tree.cpp
+++ b/simpleperf/thread_tree.cpp
@@ -231,7 +231,7 @@ const Symbol* ThreadTree::FindSymbol(const MapEntry* map, uint64_t ip,
       std::string name = android::base::StringPrintf(
           "%s%s[+%" PRIx64 "]", (show_mark_for_unknown_symbol_ ? "*" : ""),
           dso->FileName().c_str(), vaddr_in_file);
-      dso->InsertSymbol(Symbol(name, vaddr_in_file, 1));
+      dso->AddUnknownSymbol(vaddr_in_file, name);
       symbol = dso->FindSymbol(vaddr_in_file);
       CHECK(symbol != nullptr);
     } else {
@@ -256,6 +256,19 @@ void ThreadTree::ClearThreadAndMap() {
   map_storage_.clear();
 }
 
+void ThreadTree::AddDsoInfo(const std::string& file_path, uint32_t file_type,
+                            uint64_t min_vaddr, std::vector<Symbol>* symbols) {
+  DsoType dso_type = static_cast<DsoType>(file_type);
+  Dso* dso = nullptr;
+  if (dso_type == DSO_KERNEL || dso_type == DSO_KERNEL_MODULE) {
+    dso = FindKernelDsoOrNew(file_path);
+  } else {
+    dso = FindUserDsoOrNew(file_path);
+  }
+  dso->SetMinVirtualAddress(min_vaddr);
+  dso->SetSymbols(symbols);
+}
+
 void ThreadTree::Update(const Record& record) {
   if (record.type() == PERF_RECORD_MMAP) {
     const MmapRecord& r = *static_cast<const MmapRecord*>(&record);
@@ -287,21 +300,6 @@ void ThreadTree::Update(const Record& record) {
   } else if (record.type() == SIMPLE_PERF_RECORD_KERNEL_SYMBOL) {
     const auto& r = *static_cast<const KernelSymbolRecord*>(&record);
     Dso::SetKallsyms(std::move(r.kallsyms));
-  } else if (record.type() == SIMPLE_PERF_RECORD_DSO) {
-    auto& r = *static_cast<const DsoRecord*>(&record);
-    Dso* dso = nullptr;
-    if (r.dso_type == DSO_KERNEL || r.dso_type == DSO_KERNEL_MODULE) {
-      dso = FindKernelDsoOrNew(r.dso_name);
-    } else {
-      dso = FindUserDsoOrNew(r.dso_name);
-    }
-    dso->SetMinVirtualAddress(r.min_vaddr);
-    dso_id_to_dso_map_[r.dso_id] = dso;
-  } else if (record.type() == SIMPLE_PERF_RECORD_SYMBOL) {
-    auto& r = *static_cast<const SymbolRecord*>(&record);
-    Dso* dso = dso_id_to_dso_map_[r.dso_id];
-    CHECK(dso != nullptr);
-    dso->InsertSymbol(Symbol(r.name, r.addr, r.len));
   }
 }
 
diff --git a/simpleperf/thread_tree.h b/simpleperf/thread_tree.h
index 89bf059d..cd82d208 100644
--- a/simpleperf/thread_tree.h
+++ b/simpleperf/thread_tree.h
@@ -109,6 +109,9 @@ class ThreadTree {
   // the time to reload dso information.
   void ClearThreadAndMap();
 
+  void AddDsoInfo(const std::string& file_path, uint32_t file_type,
+                  uint64_t min_vaddr, std::vector<Symbol>* symbols);
+
   // Update thread tree with information provided by record.
   void Update(const Record& record);
 
@@ -135,7 +138,6 @@ class ThreadTree {
   bool show_ip_for_unknown_symbol_;
   bool show_mark_for_unknown_symbol_;
   Symbol unknown_symbol_;
-  std::unordered_map<uint64_t, Dso*> dso_id_to_dso_map_;
 };
 
 }  // namespace simpleperf