diff options
Diffstat (limited to 'pinner/pin_utils.cpp')
| -rw-r--r-- | pinner/pin_utils.cpp | 347 |
1 files changed, 347 insertions, 0 deletions
diff --git a/pinner/pin_utils.cpp b/pinner/pin_utils.cpp new file mode 100644 index 00000000..7c746bb0 --- /dev/null +++ b/pinner/pin_utils.cpp @@ -0,0 +1,347 @@ +#include "pin_utils.h" +#include <android-base/parseint.h> +#include <algorithm> +#include <fstream> +#include <map> +#include <string_view> +#include <utility> +#include <vector> + +using namespace std; +using namespace android::base; + +int write_pinlist_file(const std::string& output_file, + const std::vector<ZipEntryCoverage>& files_to_write, int64_t write_quota) { + std::vector<VmaRange> ranges; + for (auto&& file : files_to_write) { + ranges.insert(ranges.end(), file.coverage.ranges.begin(), file.coverage.ranges.end()); + } + return write_pinlist_file(output_file, ranges, write_quota); +} + +int write_pinlist_file(const std::string& output_file, const std::vector<VmaRange>& vmas_to_write, + int64_t write_quota) { + ofstream pinlist_file(output_file); + if (pinlist_file.fail()) { + return 1; + } + int64_t total_written = 0; + unsigned int page_size = sysconf(_SC_PAGESIZE); + const bool has_quota = write_quota > 0; + bool reached_quota = false; + + // The PinnerService does not require aligned offsets, however, aligning + // allows our summary results to be accurate and avoids over-accounting + // of pinning in PinnerService. + std::vector<VmaRange> processed_vmas_to_write = vmas_to_write; + align_ranges(processed_vmas_to_write, page_size); + + // When we page-align the ranges, we may cause overlaps between ranges + // as we elongate the begin offset to match the page the previous + // range may end up overlapping the current one. + processed_vmas_to_write = merge_ranges(processed_vmas_to_write); + + for (auto&& processed_vma_to_write : processed_vmas_to_write) { + uint32_t vma_start_offset = processed_vma_to_write.offset; + uint32_t vma_length = processed_vma_to_write.length; + if (has_quota && (total_written + vma_length > write_quota)) { + // We would go beyond quota, set the maximum allowed write and exit. + vma_length = write_quota - total_written; + reached_quota = true; + } + // Transform to BigEndian as PinnerService requires that endianness for reading. + uint32_t vma_start_offset_be = htobe32(vma_start_offset); + uint32_t vma_length_be = htobe32(vma_length); + cout << "Pinlist Writing start=" << vma_start_offset << " bytes=" << vma_length << endl; + pinlist_file.write(reinterpret_cast<char*>(&vma_start_offset_be), + sizeof(vma_start_offset_be)); + if (pinlist_file.fail()) { + return 1; + } + pinlist_file.write(reinterpret_cast<char*>(&vma_length_be), sizeof(vma_length_be)); + total_written += vma_length; + if (pinlist_file.fail()) { + return 1; + } + + if (reached_quota) { + break; + } + } + return 0; +} + +int read_pinlist_file(const std::string& pinner_file, /*out*/ std::vector<VmaRange>& pinranges) { + ifstream pinlist_file(pinner_file); + if (pinlist_file.fail()) { + return 1; + } + + uint32_t vma_start; + uint32_t vma_length; + while (!pinlist_file.eof()) { + pinlist_file.read(reinterpret_cast<char*>(&vma_start), sizeof(vma_start)); + pinlist_file.read(reinterpret_cast<char*>(&vma_length), sizeof(vma_length)); + if (pinlist_file.fail()) { + return 1; + } + vma_start = betoh32(vma_start); + vma_length = betoh32(vma_length); + pinranges.push_back(VmaRange(vma_start, vma_length)); + } + + return 0; +} + +ZipEntryCoverage PinConfigFile::to_zipfilemem(const ZipEntryInfo& info) { + ZipEntryCoverage file; + file.info = info; + + if (ranges.empty()) { + cout << "No ranges found for file " << info.name << " creating entire file range" << endl; + // Any file coming from pinconfig without explicit + // ranges will be assumed to be wanted in its entirety + ranges.push_back(VmaRange(0, info.file_size_bytes)); + } + + file.coverage.ranges = ranges; + + // Offsets specified in pinconfig file are relative to the file + // so transform to zip global offsets which are used for coverage + // computations. + file.coverage.apply_offset(info.offset_in_zip); + + file.coverage.compute_total_size(); + return file; +} + +int PinConfig::parse(std::string config_file, bool verbose) { + ifstream file(config_file); + string file_in_zip; + if (verbose) { + cout << "Parsing file: " << config_file << endl; + } + string token; + file >> token; + while (!file.eof()) { + if (token == "file") { + file >> file_in_zip; + PinConfigFile pin_config_file; + pin_config_file.filename = file_in_zip; + file >> token; + while (token != "file" && !file.eof()) { + VmaRange range; + // Inner parsing loop for per file config. + if (token == "offset") { + file >> token; + android::base::ParseUint(token, &range.offset); + file >> token; + if (token != "len") { + cerr << "Malformed file, expected 'len' after offset" << endl; + return 1; + } + file >> token; + android::base::ParseUint(token, &range.length); + pin_config_file.ranges.push_back(range); + } + file >> token; + } + files_.push_back(pin_config_file); + } else { + cerr << "Unexpected token: " << token << ". Exit read" << endl; + return 1; + } + } + + if (files_.empty()) { + cerr << "Failed parsing pinconfig file, no entries found." << endl; + return 1; + } + + if (verbose) { + cout << "Finished parsing Pinconfig file" << endl; + for (auto&& pin_file : files_) { + cout << "file=" << pin_file.filename << endl; + for (auto&& range : pin_file.ranges) { + cout << "offset=" << range.offset << " bytes=" << range.length << endl; + } + } + } + + return 0; +} + +void PinTool::set_custom_zip_inspector(ZipMemInspector* inspector) { + delete zip_inspector_; + zip_inspector_ = inspector; +} + +void PinTool::set_verbose_output(bool verbose) { + verbose_ = verbose; +} + +void PinTool::read_probe_from_pinlist(std::string custom_probe_file) { + custom_probe_file_ = custom_probe_file; + VmaRangeGroup* custom_probe = new VmaRangeGroup(); + read_pinlist_file(custom_probe_file_, custom_probe->ranges); + custom_probe->compute_total_size(); + if (custom_probe->ranges.empty()) { + cerr << "Did not find any memory range in " << custom_probe_file_ << endl; + delete custom_probe; + return; + } + zip_inspector_->set_existing_probe(custom_probe); +} + +int PinTool::probe_resident() { + return zip_inspector_->probe_resident(); +} + +void PinTool::compute_zip_entry_coverages() { + zip_inspector_->compute_per_file_coverage(); + if (verbose_) { + std::vector<ZipEntryInfo> files = zip_inspector_->get_file_infos(); + for (auto&& file : files) { + cout << "file found. name=" << file.name << " offset=" << file.offset_in_zip + << " uncompressed=" << file.uncompressed_size + << " compressed=" << file.file_size_bytes << endl + << endl; + } + } +} + +void PinTool::dump_coverages(PinTool::DumpType dump_type) { + std::vector<ZipEntryCoverage>* file_coverages; + if (dump_type == PinTool::DumpType::FILTERED) { + file_coverages = &filtered_files_; + } else if (dump_type == PinTool::DumpType::FILE_COVERAGE) { + file_coverages = &(zip_inspector_->get_file_coverages()); + } else { // PinTool::DumpType::PROBE + VmaRangeGroup* probe = zip_inspector_->get_probe(); + file_coverages = new vector<ZipEntryCoverage>(); + ZipEntryCoverage file; + file.coverage = *probe; + file.info.name = input_file_; + file.info.offset_in_zip = 0; + uint64_t file_size_bytes = get_file_size(input_file_); + if (file_size_bytes == -1) { + cerr << "Failed to dump, cannot fstat file: " << input_file_ << endl; + delete file_coverages; + return; + } + file.info.file_size_bytes = file_size_bytes; + file_coverages->push_back(file); + } + + for (auto&& file : *file_coverages) { + uint64_t total_size = file.coverage.compute_total_size(); + cout << file.info.name << " size(B)=" << file.info.file_size_bytes + << " resident(B)=" << total_size + << " resident(%)=" << (double)(total_size) / file.info.file_size_bytes * 100.0 << endl; + if (verbose_) { + cout << "file_base_zip_offset=" << file.info.offset_in_zip << endl; + } + cout << "file resident ranges" << endl; + if (dump_type != DumpType::PROBE) { + for (auto&& range : file.coverage.ranges) { + // The offset in the range represents the absolute absolute offset relative to the + // zip so substract the file base offset to get the relative offset within the file + // which may be what is worth for a user to specify in pinconfig.txt files. + uint64_t offset_in_file = range.offset - file.info.offset_in_zip; + + cout << "zip_offset=" << range.offset << " file_offset=" << offset_in_file + << " total_bytes=" << range.length << endl; + } + } else { + for (auto&& range : file.coverage.ranges) { + cout << "file_offset=" << range.offset << " total_bytes=" << range.length << endl; + } + } + cout << endl; + } + cout << endl; + if (dump_type == DumpType::PROBE) { + // For other dump types we do not create memory, we reuse from class. + delete file_coverages; + } +} + +void PinTool::filter_zip_entry_coverages(const std::string& pinconfig_filename) { + if (pinconfig_filename.length() == 0) { + // Nothing to do. + return; + } + + PinConfig* pinconfig = new PinConfig(); + if (pinconfig->parse(pinconfig_filename, verbose_) > 0) { + cerr << "Failed parsing pinconfig file " << pinconfig_filename << ". Skip filtering"; + delete pinconfig; + return; + } + + filter_zip_entry_coverages(pinconfig); +} + +void PinTool::filter_zip_entry_coverages(PinConfig* pinconfig) { + pinconfig_ = pinconfig; + + // Filter based on the per file configuration. + vector<ZipEntryCoverage> file_coverages = zip_inspector_->get_file_coverages(); + vector<ZipEntryCoverage>& filtered_files = filtered_files_; + + for (auto&& file_coverage : file_coverages) { + for (auto&& pinconfig_file : pinconfig_->files_) { + // Match each zip entry against every pattern in filter file. + std::string_view file_coverage_view(file_coverage.info.name.c_str()); + std::string_view pinconfig_view(pinconfig_file.filename.c_str()); + if (file_coverage_view.find(pinconfig_view) != std::string_view::npos) { + // Now that we found a match, create a file with offsets that are global to zip file + ZipEntryCoverage file_in_config = pinconfig_file.to_zipfilemem(file_coverage.info); + if (verbose_) { + cout << "Found a match: file=" << file_coverage.info.name + << " matching filter=" << pinconfig_file.filename << endl; + for (auto&& range : file_in_config.coverage.ranges) { + cout << "zip_offset=" << range.offset << " bytes=" << range.length << endl; + } + } + ZipEntryCoverage filtered_file = + file_coverage.compute_coverage(file_in_config.coverage); + filtered_files.push_back(filtered_file); + break; + } + } + } +} + +std::vector<ZipEntryCoverage> PinTool::get_filtered_zip_entries() { + return filtered_files_; +} + +void PinTool::write_coverages_as_pinlist(std::string output_pinlist, int64_t write_quota) { + std::vector<ZipEntryCoverage>* pinlist_coverages = nullptr; + if (!filtered_files_.empty()) { + // Highest preference is writing filtered files if they exist + if (verbose_) { + cout << "Writing pinconfig filtered file coverages" << endl; + } + pinlist_coverages = &filtered_files_; + } else if (!zip_inspector_->get_file_coverages().empty()) { + // Fallback to looking for file coverage computation + pinlist_coverages = &zip_inspector_->get_file_coverages(); + if (verbose_) { + cout << "Writing regular file coverages." << endl; + } + } + if (pinlist_coverages == nullptr) { + cerr << "Failed to find coverage to write to: " << output_pinlist << endl; + return; + } + int res = write_pinlist_file(output_pinlist, *pinlist_coverages, write_quota); + if (res > 0) { + cerr << "Failed to write pin file at: " << output_pinlist << endl; + } else { + if (verbose_) { + cout << "Finished writing pin file at: " << output_pinlist << endl; + } + } +}
\ No newline at end of file |