#!/usr/bin/python2 # Copyright 2016 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Processes the functions from the pprof(go/pprof) files and CWP(go/cwp) data. The pprof --top and pprof --tree outputs should be extracted from the benchmark profiles. The outputs contain the hot functions and the call chains. For each pair of pprof --top and --tree output files, the tool will create a file that contains the hot functions present also in the extracted CWP data. The common functions are organized in groups that represent a Chrome OS component. A function belongs to a group that is defined by a given file path if it is declared in a file that shares that path. A set of metrics are computed for each function, benchmark and Chrome OS group covered by a benchmark. Afterwards, this script extracts the functions that are present in the CWP data and not in the benchmark profiles. The extra functions are also groupped in Chrome OS components. """ from collections import defaultdict import argparse import os import shutil import sys import benchmark_metrics import utils class HotFunctionsProcessor(object): """Does the pprof and CWP output processing. Extracts the common, extra functions from the pprof files, groups them in Chrome OS components. Computes the metrics for the common functions, benchmark and Chrome OS groups covered by a benchmark. """ def __init__(self, pprof_top_path, pprof_tree_path, cwp_inclusive_count_file, cwp_pairwise_inclusive_count_file, cwp_function_groups_file, common_functions_path, common_functions_groups_path, benchmark_set_metrics_file, extra_cwp_functions_file, extra_cwp_functions_groups_file, extra_cwp_functions_groups_path): """Initializes the HotFunctionsProcessor. Args: pprof_top_path: The directory containing the files with the pprof --top output. pprof_tree_path: The directory containing the files with the pprof --tree output. cwp_inclusive_count_file: The CSV file containing the CWP functions with the inclusive count values. cwp_pairwise_inclusive_count_file: The CSV file containing the CWP pairs of parent and child functions with their inclusive count values. cwp_function_groups_file: The file that contains the CWP function groups. common_functions_path: The directory containing the CSV output files with the common functions of the benchmark profiles and CWP data. common_functions_groups_path: The directory containing the CSV output files with the CWP groups and their metrics that match the common functions of the benchmark profiles and CWP. benchmark_set_metrics_file: The CSV output file containing the metrics for each benchmark. extra_cwp_functions_file: The CSV output file containing the functions that are in the CWP data, but are not in any of the benchmark profiles. extra_cwp_functions_groups_file: The CSV output file containing the groups that match the extra CWP functions and their statistics. extra_cwp_functions_groups_path: The directory containing the CSV output files with the extra CWP functions that match a particular group. """ self._pprof_top_path = pprof_top_path self._pprof_tree_path = pprof_tree_path self._cwp_inclusive_count_file = cwp_inclusive_count_file self._cwp_pairwise_inclusive_count_file = cwp_pairwise_inclusive_count_file self._cwp_function_groups_file = cwp_function_groups_file self._common_functions_path = common_functions_path self._common_functions_groups_path = common_functions_groups_path self._benchmark_set_metrics_file = benchmark_set_metrics_file self._extra_cwp_functions_file = extra_cwp_functions_file self._extra_cwp_functions_groups_file = extra_cwp_functions_groups_file self._extra_cwp_functions_groups_path = extra_cwp_functions_groups_path def ProcessHotFunctions(self): """Does the processing of the hot functions.""" with open(self._cwp_function_groups_file) as input_file: cwp_function_groups = utils.ParseFunctionGroups(input_file.readlines()) cwp_statistics = \ self.ExtractCommonFunctions(self._pprof_top_path, self._pprof_tree_path, self._cwp_inclusive_count_file, self._cwp_pairwise_inclusive_count_file, cwp_function_groups, self._common_functions_path, self._common_functions_groups_path, self._benchmark_set_metrics_file) self.ExtractExtraFunctions(cwp_statistics, self._extra_cwp_functions_file) self.GroupExtraFunctions(cwp_statistics, cwp_function_groups, self._extra_cwp_functions_groups_path, self._extra_cwp_functions_groups_file) def ExtractCommonFunctions(self, pprof_top_path, pprof_tree_path, cwp_inclusive_count_file, cwp_pairwise_inclusive_count_file, cwp_function_groups, common_functions_path, common_functions_groups_path, benchmark_set_metrics_file): """Extracts the common functions of the benchmark profiles and the CWP data. For each pair of pprof --top and --tree output files, it creates a separate file with the same name containing the common functions specifications and metrics, that will be placed in the common_functions_path directory. The resulting file is in CSV format, containing the following fields: function name, file name, object, inclusive count, inclusive_count_fraction, flat, flat%, sum%, cum, cum%, distance and score. For each pair of pprof files, an additional file is created with the Chrome OS groups that match the common functions. The file is in CSV format containing the fields: group name, group path, the number of functions that match the group, the average and cumulative distance, the average and cumulative score. The file has the same name with the pprof file and it is placed in the common_functions_groups_path directory. For all the analyzed benchmarks, the method creates a CSV output file containing the metrics for each benchmark. The CSV fields include the benchmark name, the number of common functions, the average and cumulative distance and score. It builds a dict of the CWP statistics by calling the utils.ParseCWPInclusiveCountFile method and if a function is common, it is marked as a COMMON_FUNCTION. Args: pprof_top_path: The name of the directory with the files with the pprof --top output. pprof_tree_path: The name of the directory with the files with the pprof --tree output. cwp_inclusive_count_file: A dict with the inclusive count values. cwp_pairwise_inclusive_count_file: A dict with the pairwise inclusive count values. cwp_function_groups: A list of tuples containing the name of the group and the corresponding file path. common_functions_path: The path containing the output files with the common functions and their metrics. common_functions_groups_path: The path containing the output files with the Chrome OS groups that match the common functions and their metrics. benchmark_set_metrics_file: The CSV output file containing the metrics for all the analyzed benchmarks. Returns: A dict containing the CWP statistics with the common functions marked as COMMON_FUNCTION. """ cwp_inclusive_count_statistics = \ utils.ParseCWPInclusiveCountFile(cwp_inclusive_count_file) cwp_pairwise_inclusive_count_statistics = \ utils.ParseCWPPairwiseInclusiveCountFile( cwp_pairwise_inclusive_count_file) cwp_inclusive_count_statistics_cumulative = \ utils.ComputeCWPCummulativeInclusiveStatistics( cwp_inclusive_count_statistics) cwp_pairwise_inclusive_count_fractions = \ utils.ComputeCWPChildFunctionsFractions( cwp_inclusive_count_statistics_cumulative, cwp_pairwise_inclusive_count_statistics) benchmark_set_metrics = {} pprof_files = os.listdir(pprof_top_path) for pprof_file in pprof_files: pprof_top_statistics = \ utils.ParsePprofTopOutput(os.path.join(pprof_top_path, pprof_file)) pprof_tree_statistics = \ utils.ParsePprofTreeOutput(os.path.join(pprof_tree_path, pprof_file)) common_functions_lines = [] benchmark_function_metrics = {} for function_key, function_statistic in pprof_top_statistics.iteritems(): if function_key not in cwp_inclusive_count_statistics: continue cwp_dso_name, cwp_inclusive_count, cwp_inclusive_count_fraction, _ = \ cwp_inclusive_count_statistics[function_key] cwp_inclusive_count_statistics[function_key] = \ (cwp_dso_name, cwp_inclusive_count, cwp_inclusive_count_fraction, utils.COMMON_FUNCTION) function_name, _ = function_key.split(',') distance = benchmark_metrics.ComputeDistanceForFunction( pprof_tree_statistics[function_key], cwp_pairwise_inclusive_count_fractions.get(function_name, {})) benchmark_cum_p = float(function_statistic[4]) score = benchmark_metrics.ComputeScoreForFunction( distance, cwp_inclusive_count_fraction, benchmark_cum_p) benchmark_function_metrics[function_key] = (distance, score) common_functions_lines.append(','.join([function_key, cwp_dso_name, str( cwp_inclusive_count), str(cwp_inclusive_count_fraction), ','.join( function_statistic), str(distance), str(score)])) benchmark_function_groups_statistics = \ benchmark_metrics.ComputeMetricsForComponents( cwp_function_groups, benchmark_function_metrics) benchmark_set_metrics[pprof_file] = \ benchmark_metrics.ComputeMetricsForBenchmark( benchmark_function_metrics) with open(os.path.join(common_functions_path, pprof_file), 'w') \ as output_file: common_functions_lines.sort( key=lambda x: float(x.split(',')[11]), reverse=True) common_functions_lines.insert(0, 'function,file,dso,inclusive_count,' 'inclusive_count_fraction,flat,flat%,' 'sum%,cum,cum%,distance,score') output_file.write('\n'.join(common_functions_lines)) with open(os.path.join(common_functions_groups_path, pprof_file), 'w') \ as output_file: common_functions_groups_lines = \ [','.join([group_name, ','.join( [str(statistic) for statistic in group_statistic])]) for group_name, group_statistic in benchmark_function_groups_statistics.iteritems()] common_functions_groups_lines.sort( key=lambda x: float(x.split(',')[5]), reverse=True) common_functions_groups_lines.insert( 0, 'group_name,file_path,number_of_functions,distance_cum,' 'distance_avg,score_cum,score_avg') output_file.write('\n'.join(common_functions_groups_lines)) with open(benchmark_set_metrics_file, 'w') as output_file: benchmark_set_metrics_lines = [] for benchmark_name, metrics in benchmark_set_metrics.iteritems(): benchmark_set_metrics_lines.append(','.join([benchmark_name, ','.join( [str(metric) for metric in metrics])])) benchmark_set_metrics_lines.sort( key=lambda x: float(x.split(',')[4]), reverse=True) benchmark_set_metrics_lines.insert( 0, 'benchmark_name,number_of_functions,distance_cum,distance_avg,' 'score_cum,score_avg') output_file.write('\n'.join(benchmark_set_metrics_lines)) return cwp_inclusive_count_statistics def GroupExtraFunctions(self, cwp_statistics, cwp_function_groups, extra_cwp_functions_groups_path, extra_cwp_functions_groups_file): """Groups the extra functions. Writes the data of the functions that belong to each group in a separate file, sorted by their inclusive count value, in descending order. The file name is the same as the group name. The file is in CSV format, containing the fields: function name, file name, object name, inclusive count, inclusive count fraction. It creates a CSV file containing the name of the group, their common path, the total inclusive count and inclusive count fraction values of all the functions declared in files that share the common path, sorted in descending order by the inclusive count value. Args: cwp_statistics: A dict containing the CWP statistics. cwp_function_groups: A list of tuples with the groups names and the path describing the groups. extra_cwp_functions_groups_path: The name of the directory containing the CSV output files with the extra CWP functions that match a particular group. extra_cwp_functions_groups_file: The CSV output file containing the groups that match the extra functions and their statistics. """ cwp_function_groups_statistics = defaultdict(lambda: ([], '', 0, 0.0)) for function, statistics in cwp_statistics.iteritems(): if statistics[3] == utils.COMMON_FUNCTION: continue file_name = function.split(',')[1] group_inclusive_count = int(statistics[1]) group_inclusive_count_fraction = float(statistics[2]) for group in cwp_function_groups: group_common_path = group[1] if group_common_path not in file_name: continue group_name = group[0] group_statistics = cwp_function_groups_statistics[group_name] group_lines = group_statistics[0] group_inclusive_count += group_statistics[2] group_inclusive_count_fraction += group_statistics[3] group_lines.append(','.join([function, statistics[0], str(statistics[1]), str(statistics[2])])) cwp_function_groups_statistics[group_name] = \ (group_lines, group_common_path, group_inclusive_count, group_inclusive_count_fraction) break extra_cwp_functions_groups_lines = [] for group_name, group_statistics \ in cwp_function_groups_statistics.iteritems(): group_output_lines = group_statistics[0] group_output_lines.sort(key=lambda x: int(x.split(',')[3]), reverse=True) group_output_lines.insert( 0, 'function,file,dso,inclusive_count,inclusive_count_fraction') with open(os.path.join(extra_cwp_functions_groups_path, group_name), 'w') as output_file: output_file.write('\n'.join(group_output_lines)) extra_cwp_functions_groups_lines.append(','.join( [group_name, group_statistics[1], str(group_statistics[2]), str( group_statistics[3])])) extra_cwp_functions_groups_lines.sort( key=lambda x: int(x.split(',')[2]), reverse=True) extra_cwp_functions_groups_lines.insert( 0, 'group,shared_path,inclusive_count,inclusive_count_fraction') with open(extra_cwp_functions_groups_file, 'w') as output_file: output_file.write('\n'.join(extra_cwp_functions_groups_lines)) def ExtractExtraFunctions(self, cwp_statistics, extra_cwp_functions_file): """Gets the functions that are in the CWP data, but not in the pprof output. Writes the functions and their statistics in the extra_cwp_functions_file file. The output is sorted based on the inclusive_count value. The file is in CSV format, containing the fields: function name, file name, object name, inclusive count and inclusive count fraction. Args: cwp_statistics: A dict containing the CWP statistics indexed by the function and the file name, comma separated. extra_cwp_functions_file: The file where it should be stored the CWP functions and statistics that are marked as EXTRA_FUNCTION. """ output_lines = [] for function, statistics in cwp_statistics.iteritems(): if statistics[3] == utils.EXTRA_FUNCTION: output_lines.append(','.join([function, statistics[0], str(statistics[1]), str(statistics[2])])) with open(extra_cwp_functions_file, 'w') as output_file: output_lines.sort(key=lambda x: int(x.split(',')[3]), reverse=True) output_lines.insert(0, 'function,file,dso,inclusive_count,' 'inclusive_count_fraction') output_file.write('\n'.join(output_lines)) def ParseArguments(arguments): parser = argparse.ArgumentParser() parser.add_argument( '--pprof_top_path', required=True, help='The directory containing the files with the pprof --top output of ' 'the benchmark profiles (the hot functions). The name of the files ' 'should match with the ones from the pprof tree output files.') parser.add_argument( '--pprof_tree_path', required=True, help='The directory containing the files with the pprof --tree output ' 'of the benchmark profiles (the call chains). The name of the files ' 'should match with the ones of the pprof top output files.') parser.add_argument( '--cwp_inclusive_count_file', required=True, help='The CSV file containing the CWP hot functions with their ' 'inclusive_count values. The CSV fields include the name of the ' 'function, the file and the object with the definition, the inclusive ' 'count value and the inclusive count fraction out of the total amount of ' 'inclusive count values.') parser.add_argument( '--cwp_pairwise_inclusive_count_file', required=True, help='The CSV file containing the CWP pairs of parent and child ' 'functions with their inclusive count values. The CSV fields include the ' 'name of the parent and child functions concatenated by ;;, the file ' 'and the object with the definition of the child function, and the ' 'inclusive count value.') parser.add_argument( '--cwp_function_groups_file', required=True, help='The file that contains the CWP function groups. A line consists in ' 'the group name and a file path describing the group. A group must ' 'represent a ChromeOS component.') parser.add_argument( '--common_functions_path', required=True, help='The directory containing the CSV output files with the common ' 'functions of the benchmark profiles and CWP data. A file will contain ' 'all the hot functions from a pprof top output file that are also ' 'included in the file containing the cwp inclusive count values. The CSV ' 'fields are: the function name, the file and the object where the ' 'function is declared, the CWP inclusive count and inclusive count ' 'fraction values, the cumulative and average distance, the cumulative ' 'and average score. The files with the common functions will have the ' 'same names with the corresponding pprof output files.') parser.add_argument( '--common_functions_groups_path', required=True, help='The directory containing the CSV output files with the Chrome OS ' 'groups and their metrics that match the common functions of the ' 'benchmark profiles and CWP. The files with the groups will have the ' 'same names with the corresponding pprof output files. The CSV fields ' 'include the group name, group path, the number of functions that match ' 'the group, the average and cumulative distance, the average and ' 'cumulative score.') parser.add_argument( '--benchmark_set_metrics_file', required=True, help='The CSV output file containing the metrics for each benchmark. The ' 'CSV fields include the benchmark name, the number of common functions, ' 'the average and cumulative distance and score.') parser.add_argument( '--extra_cwp_functions_file', required=True, help='The CSV output file containing the functions that are in the CWP ' 'data, but are not in any of the benchmark profiles. The CSV fields ' 'include the name of the function, the file name and the object with the ' 'definition, and the CWP inclusive count and inclusive count fraction ' 'values. The entries are sorted in descending order based on the ' 'inclusive count value.') parser.add_argument( '--extra_cwp_functions_groups_file', required=True, help='The CSV output file containing the groups that match the extra CWP ' 'functions and their statistics. The CSV fields include the group name, ' 'the file path, the total inclusive count and inclusive count fraction ' 'values of the functions matching a particular group.') parser.add_argument( '--extra_cwp_functions_groups_path', required=True, help='The directory containing the CSV output files with the extra CWP ' 'functions that match a particular group. The name of the file is the ' 'same as the group name. The CSV fields include the name of the ' 'function, the file name and the object with the definition, and the CWP ' 'inclusive count and inclusive count fraction values. The entries are ' 'sorted in descending order based on the inclusive count value.') options = parser.parse_args(arguments) return options def Main(argv): options = ParseArguments(argv) if os.path.exists(options.common_functions_path): shutil.rmtree(options.common_functions_path) os.makedirs(options.common_functions_path) if os.path.exists(options.common_functions_groups_path): shutil.rmtree(options.common_functions_groups_path) os.makedirs(options.common_functions_groups_path) if os.path.exists(options.extra_cwp_functions_groups_path): shutil.rmtree(options.extra_cwp_functions_groups_path) os.makedirs(options.extra_cwp_functions_groups_path) hot_functions_processor = HotFunctionsProcessor( options.pprof_top_path, options.pprof_tree_path, options.cwp_inclusive_count_file, options.cwp_pairwise_inclusive_count_file, options.cwp_function_groups_file, options.common_functions_path, options.common_functions_groups_path, options.benchmark_set_metrics_file, options.extra_cwp_functions_file, options.extra_cwp_functions_groups_file, options.extra_cwp_functions_groups_path) hot_functions_processor.ProcessHotFunctions() if __name__ == '__main__': Main(sys.argv[1:])