path: root/user_activity_benchmarks/process_hot_functions.py

#!/usr/bin/python2

# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Processes the functions from the pprof(go/pprof) files and CWP(go/cwp) data.

The pprof --top and pprof --tree outputs should be extracted from the benchmark
profiles. The outputs contain the hot functions and the call chains.

For each pair of pprof --top and --tree output files, the tool will create a
file that contains the hot functions present also in the extracted CWP data.
The common functions are organized in groups that represent a Chrome OS
component. A function belongs to a group that is defined by a given file path
if it is declared in a file that shares that path.

A set of metrics are computed for each function, benchmark and Chrome OS group
covered by a benchmark.

Afterwards, this script extracts the functions that are present in the CWP
data and not in the benchmark profiles. The extra functions are also groupped
in Chrome OS components.
"""

from collections import defaultdict

import argparse
import os
import shutil
import sys

import benchmark_metrics
import utils


class HotFunctionsProcessor(object):
  """Does the pprof and CWP output processing.

  Extracts the common, extra functions from the pprof files, groups them in
  Chrome OS components. Computes the metrics for the common functions,
  benchmark and Chrome OS groups covered by a benchmark.
  """

  def __init__(self, pprof_top_path, pprof_tree_path, cwp_inclusive_count_file,
               cwp_pairwise_inclusive_count_file, cwp_function_groups_file,
               common_functions_path, common_functions_groups_path,
               benchmark_set_metrics_file, extra_cwp_functions_file,
               extra_cwp_functions_groups_file,
               extra_cwp_functions_groups_path):
    """Initializes the HotFunctionsProcessor.

    Args:
      pprof_top_path: The directory containing the files with the pprof --top
        output.
      pprof_tree_path: The directory containing the files with the pprof --tree
        output.
      cwp_inclusive_count_file: The CSV file containing the CWP functions with
        the inclusive count values.
      cwp_pairwise_inclusive_count_file: The CSV file containing the CWP pairs
        of parent and child functions with their inclusive count values.
      cwp_function_groups_file: The file that contains the CWP function groups.
      common_functions_path: The directory containing the CSV output files
        with the common functions of the benchmark profiles and CWP data.
      common_functions_groups_path: The directory containing the CSV output
        files with the CWP groups and their metrics that match the common
        functions of the benchmark profiles and CWP.
      benchmark_set_metrics_file: The CSV output file containing the metrics for
        each benchmark.
      extra_cwp_functions_file: The CSV output file containing the functions
        that are in the CWP data, but are not in any of the benchmark profiles.
      extra_cwp_functions_groups_file: The CSV output file containing the groups
        that match the extra CWP functions and their statistics.
      extra_cwp_functions_groups_path: The directory containing the CSV output
        files with the extra CWP functions that match a particular group.
    """
    self._pprof_top_path = pprof_top_path
    self._pprof_tree_path = pprof_tree_path
    self._cwp_inclusive_count_file = cwp_inclusive_count_file
    self._cwp_pairwise_inclusive_count_file = cwp_pairwise_inclusive_count_file
    self._cwp_function_groups_file = cwp_function_groups_file
    self._common_functions_path = common_functions_path
    self._common_functions_groups_path = common_functions_groups_path
    self._benchmark_set_metrics_file = benchmark_set_metrics_file
    self._extra_cwp_functions_file = extra_cwp_functions_file
    self._extra_cwp_functions_groups_file = extra_cwp_functions_groups_file
    self._extra_cwp_functions_groups_path = extra_cwp_functions_groups_path

  def ProcessHotFunctions(self):
    """Does the processing of the hot functions."""
    with open(self._cwp_function_groups_file) as input_file:
      cwp_function_groups = utils.ParseFunctionGroups(input_file.readlines())
    cwp_statistics = \
      self.ExtractCommonFunctions(self._pprof_top_path,
                                  self._pprof_tree_path,
                                  self._cwp_inclusive_count_file,
                                  self._cwp_pairwise_inclusive_count_file,
                                  cwp_function_groups,
                                  self._common_functions_path,
                                  self._common_functions_groups_path,
                                  self._benchmark_set_metrics_file)
    self.ExtractExtraFunctions(cwp_statistics, self._extra_cwp_functions_file)
    self.GroupExtraFunctions(cwp_statistics, cwp_function_groups,
                             self._extra_cwp_functions_groups_path,
                             self._extra_cwp_functions_groups_file)

  def ExtractCommonFunctions(self, pprof_top_path, pprof_tree_path,
                             cwp_inclusive_count_file,
                             cwp_pairwise_inclusive_count_file,
                             cwp_function_groups, common_functions_path,
                             common_functions_groups_path,
                             benchmark_set_metrics_file):
    """Extracts the common functions of the benchmark profiles and the CWP data.

    For each pair of pprof --top and --tree output files, it creates a separate
    file with the same name containing the common functions specifications and
    metrics, that will be placed in the common_functions_path directory.

    The resulting file is in CSV format, containing the following fields:
    function name, file name, object, inclusive count, inclusive_count_fraction,
    flat, flat%, sum%, cum, cum%, distance and score.

    For each pair of pprof files, an additional file is created with the
    Chrome OS groups that match the common functions.

    The file is in CSV format containing the fields: group name, group path,
    the number of functions that match the group, the average and cumulative
    distance, the average and cumulative score.
    The file has the same name with the pprof file and it is placed in the
    common_functions_groups_path directory.

    For all the analyzed benchmarks, the method creates a CSV output file
    containing the metrics for each benchmark. The CSV fields include the
    benchmark name, the number of common functions, the average and
    cumulative distance and score.

    It builds a dict of the CWP statistics by calling the
    utils.ParseCWPInclusiveCountFile method and if a function is common, it is
    marked as a COMMON_FUNCTION.

    Args:
      pprof_top_path: The name of the directory with the files with the
        pprof --top output.
      pprof_tree_path: The name of the directory with the files with the
        pprof --tree output.
      cwp_inclusive_count_file: A dict with the inclusive count values.
      cwp_pairwise_inclusive_count_file: A dict with the pairwise inclusive
        count values.
      cwp_function_groups: A list of tuples containing the name of the group
        and the corresponding file path.
      common_functions_path: The path containing the output files with the
        common functions and their metrics.
      common_functions_groups_path: The path containing the output files with
        the Chrome OS groups that match the common functions and their metrics.
      benchmark_set_metrics_file: The CSV output file containing the metrics for
        all the analyzed benchmarks.

    Returns:
      A dict containing the CWP statistics with the common functions marked as
      COMMON_FUNCTION.
    """
    cwp_inclusive_count_statistics = \
        utils.ParseCWPInclusiveCountFile(cwp_inclusive_count_file)
    cwp_pairwise_inclusive_count_statistics = \
        utils.ParseCWPPairwiseInclusiveCountFile(
            cwp_pairwise_inclusive_count_file)
    cwp_inclusive_count_statistics_cumulative = \
        utils.ComputeCWPCummulativeInclusiveStatistics(
            cwp_inclusive_count_statistics)
    cwp_pairwise_inclusive_count_fractions = \
        utils.ComputeCWPChildFunctionsFractions(
            cwp_inclusive_count_statistics_cumulative,
            cwp_pairwise_inclusive_count_statistics)
    benchmark_set_metrics = {}
    pprof_files = os.listdir(pprof_top_path)

    for pprof_file in pprof_files:
      pprof_top_statistics = \
          utils.ParsePprofTopOutput(os.path.join(pprof_top_path, pprof_file))
      pprof_tree_statistics = \
          utils.ParsePprofTreeOutput(os.path.join(pprof_tree_path, pprof_file))
      common_functions_lines = []
      benchmark_function_metrics = {}

      for function_key, function_statistic in pprof_top_statistics.iteritems():
        if function_key not in cwp_inclusive_count_statistics:
          continue

        cwp_dso_name, cwp_inclusive_count, cwp_inclusive_count_fraction, _ = \
            cwp_inclusive_count_statistics[function_key]
        cwp_inclusive_count_statistics[function_key] = \
            (cwp_dso_name, cwp_inclusive_count, cwp_inclusive_count_fraction,
             utils.COMMON_FUNCTION)

        function_name, _ = function_key.split(',')
        distance = benchmark_metrics.ComputeDistanceForFunction(
            pprof_tree_statistics[function_key],
            cwp_pairwise_inclusive_count_fractions.get(function_name, {}))
        benchmark_cum_p = float(function_statistic[4])
        score = benchmark_metrics.ComputeScoreForFunction(
            distance, cwp_inclusive_count_fraction, benchmark_cum_p)
        benchmark_function_metrics[function_key] = (distance, score)

        common_functions_lines.append(','.join([function_key, cwp_dso_name, str(
            cwp_inclusive_count), str(cwp_inclusive_count_fraction), ','.join(
                function_statistic), str(distance), str(score)]))
      benchmark_function_groups_statistics = \
          benchmark_metrics.ComputeMetricsForComponents(
              cwp_function_groups, benchmark_function_metrics)
      benchmark_set_metrics[pprof_file] = \
          benchmark_metrics.ComputeMetricsForBenchmark(
              benchmark_function_metrics)

      with open(os.path.join(common_functions_path, pprof_file), 'w') \
          as output_file:
        common_functions_lines.sort(
            key=lambda x: float(x.split(',')[11]), reverse=True)
        common_functions_lines.insert(0, 'function,file,dso,inclusive_count,'
                                      'inclusive_count_fraction,flat,flat%,'
                                      'sum%,cum,cum%,distance,score')
        output_file.write('\n'.join(common_functions_lines))

      with open(os.path.join(common_functions_groups_path, pprof_file), 'w') \
          as output_file:
        common_functions_groups_lines = \
            [','.join([group_name, ','.join(
                [str(statistic) for statistic in group_statistic])])
             for group_name, group_statistic in
             benchmark_function_groups_statistics.iteritems()]
        common_functions_groups_lines.sort(
            key=lambda x: float(x.split(',')[5]), reverse=True)
        common_functions_groups_lines.insert(
            0, 'group_name,file_path,number_of_functions,distance_cum,'
            'distance_avg,score_cum,score_avg')
        output_file.write('\n'.join(common_functions_groups_lines))

    with open(benchmark_set_metrics_file, 'w') as output_file:
      benchmark_set_metrics_lines = []

      for benchmark_name, metrics in benchmark_set_metrics.iteritems():
        benchmark_set_metrics_lines.append(','.join([benchmark_name, ','.join(
            [str(metric) for metric in metrics])]))
      benchmark_set_metrics_lines.sort(
          key=lambda x: float(x.split(',')[4]), reverse=True)
      benchmark_set_metrics_lines.insert(
          0, 'benchmark_name,number_of_functions,distance_cum,distance_avg,'
          'score_cum,score_avg')
      output_file.write('\n'.join(benchmark_set_metrics_lines))

    return cwp_inclusive_count_statistics

  def GroupExtraFunctions(self, cwp_statistics, cwp_function_groups,
                          extra_cwp_functions_groups_path,
                          extra_cwp_functions_groups_file):
    """Groups the extra functions.

    Writes the data of the functions that belong to each group in a separate
    file, sorted by their inclusive count value, in descending order. The file
    name is the same as the group name.

    The file is in CSV format, containing the fields: function name, file name,
    object name, inclusive count, inclusive count fraction.

    It creates a CSV file containing the name of the group, their
    common path, the total inclusive count and inclusive count fraction values
    of all the functions declared in files that share the common path, sorted
    in descending order by the inclusive count value.

    Args:
      cwp_statistics: A dict containing the CWP statistics.
      cwp_function_groups: A list of tuples with the groups names and the path
        describing the groups.
      extra_cwp_functions_groups_path: The name of the directory containing
        the CSV output files with the extra CWP functions that match a
        particular group.
      extra_cwp_functions_groups_file: The CSV output file containing the groups
        that match the extra functions and their statistics.
    """
    cwp_function_groups_statistics = defaultdict(lambda: ([], '', 0, 0.0))
    for function, statistics in cwp_statistics.iteritems():
      if statistics[3] == utils.COMMON_FUNCTION:
        continue

      file_name = function.split(',')[1]
      group_inclusive_count = int(statistics[1])
      group_inclusive_count_fraction = float(statistics[2])

      for group in cwp_function_groups:
        group_common_path = group[1]

        if group_common_path not in file_name:
          continue

        group_name = group[0]
        group_statistics = cwp_function_groups_statistics[group_name]
        group_lines = group_statistics[0]
        group_inclusive_count += group_statistics[2]
        group_inclusive_count_fraction += group_statistics[3]

        group_lines.append(','.join([function, statistics[0],
                                     str(statistics[1]), str(statistics[2])]))
        cwp_function_groups_statistics[group_name] = \
            (group_lines, group_common_path, group_inclusive_count,
             group_inclusive_count_fraction)
        break

    extra_cwp_functions_groups_lines = []
    for group_name, group_statistics \
        in cwp_function_groups_statistics.iteritems():
      group_output_lines = group_statistics[0]
      group_output_lines.sort(key=lambda x: int(x.split(',')[3]), reverse=True)
      group_output_lines.insert(
          0, 'function,file,dso,inclusive_count,inclusive_count_fraction')
      with open(os.path.join(extra_cwp_functions_groups_path, group_name),
                'w') as output_file:
        output_file.write('\n'.join(group_output_lines))
      extra_cwp_functions_groups_lines.append(','.join(
          [group_name, group_statistics[1], str(group_statistics[2]), str(
              group_statistics[3])]))

    extra_cwp_functions_groups_lines.sort(
        key=lambda x: int(x.split(',')[2]), reverse=True)
    extra_cwp_functions_groups_lines.insert(
        0, 'group,shared_path,inclusive_count,inclusive_count_fraction')
    with open(extra_cwp_functions_groups_file, 'w') as output_file:
      output_file.write('\n'.join(extra_cwp_functions_groups_lines))

  def ExtractExtraFunctions(self, cwp_statistics, extra_cwp_functions_file):
    """Gets the functions that are in the CWP data, but not in the pprof output.

    Writes the functions and their statistics in the extra_cwp_functions_file
    file. The output is sorted based on the inclusive_count value. The file is
    in CSV format, containing the fields: function name, file name, object name,
    inclusive count and inclusive count fraction.

    Args:
      cwp_statistics: A dict containing the CWP statistics indexed by the
        function and the file name, comma separated.
      extra_cwp_functions_file: The file where it should be stored the CWP
        functions and statistics that are marked as EXTRA_FUNCTION.
    """
    output_lines = []

    for function, statistics in cwp_statistics.iteritems():
      if statistics[3] == utils.EXTRA_FUNCTION:
        output_lines.append(','.join([function, statistics[0],
                                      str(statistics[1]), str(statistics[2])]))

    with open(extra_cwp_functions_file, 'w') as output_file:
      output_lines.sort(key=lambda x: int(x.split(',')[3]), reverse=True)
      output_lines.insert(0, 'function,file,dso,inclusive_count,'
                          'inclusive_count_fraction')
      output_file.write('\n'.join(output_lines))


def ParseArguments(arguments):
  parser = argparse.ArgumentParser()

  parser.add_argument(
      '--pprof_top_path',
      required=True,
      help='The directory containing the files with the pprof --top output of '
      'the benchmark profiles (the hot functions). The name of the files '
      'should match with the ones from the pprof tree output files.')
  parser.add_argument(
      '--pprof_tree_path',
      required=True,
      help='The directory containing the files with the pprof --tree output '
      'of the benchmark profiles (the call chains). The name of the files '
      'should match with the ones of the pprof top output files.')
  parser.add_argument(
      '--cwp_inclusive_count_file',
      required=True,
      help='The CSV file containing the CWP hot functions with their '
      'inclusive_count values. The CSV fields include the name of the '
      'function, the file and the object with the definition, the inclusive '
      'count value and the inclusive count fraction out of the total amount of '
      'inclusive count values.')
  parser.add_argument(
      '--cwp_pairwise_inclusive_count_file',
      required=True,
      help='The CSV file containing the CWP pairs of parent and child '
      'functions with their inclusive count values. The CSV fields include the '
      'name of the parent and child functions concatenated by ;;, the file '
      'and the object with the definition of the child function, and the '
      'inclusive count value.')
  parser.add_argument(
      '--cwp_function_groups_file',
      required=True,
      help='The file that contains the CWP function groups. A line consists in '
      'the group name and a file path describing the group. A group must '
      'represent a ChromeOS component.')
  parser.add_argument(
      '--common_functions_path',
      required=True,
      help='The directory containing the CSV output files with the common '
      'functions of the benchmark profiles and CWP data. A file will contain '
      'all the hot functions from a pprof top output file that are also '
      'included in the file containing the cwp inclusive count values. The CSV '
      'fields are: the function name, the file and the object where the '
      'function is declared, the CWP inclusive count and inclusive count '
      'fraction values, the cumulative and average distance, the cumulative '
      'and average score. The files with the common functions will have the '
      'same names with the corresponding pprof output files.')
  parser.add_argument(
      '--common_functions_groups_path',
      required=True,
      help='The directory containing the CSV output files with the Chrome OS '
      'groups and their metrics that match the common functions of the '
      'benchmark profiles and CWP. The files with the groups will have the '
      'same names with the corresponding pprof output files. The CSV fields '
      'include the group name, group path, the number of functions that match '
      'the group, the average and cumulative distance, the average and '
      'cumulative score.')
  parser.add_argument(
      '--benchmark_set_metrics_file',
      required=True,
      help='The CSV output file containing the metrics for each benchmark. The '
      'CSV fields include the benchmark name, the number of common functions, '
      'the average and cumulative distance and score.')
  parser.add_argument(
      '--extra_cwp_functions_file',
      required=True,
      help='The CSV output file containing the functions that are in the CWP '
      'data, but are not in any of the benchmark profiles. The CSV fields '
      'include the name of the function, the file name and the object with the '
      'definition, and the CWP inclusive count and inclusive count fraction '
      'values. The entries are sorted in descending order based on the '
      'inclusive count value.')
  parser.add_argument(
      '--extra_cwp_functions_groups_file',
      required=True,
      help='The CSV output file containing the groups that match the extra CWP '
      'functions and their statistics. The CSV fields include the group name, '
      'the file path, the total inclusive count and inclusive count fraction '
      'values of the functions matching a particular group.')
  parser.add_argument(
      '--extra_cwp_functions_groups_path',
      required=True,
      help='The directory containing the CSV output files with the extra CWP '
      'functions that match a particular group. The name of the file is the '
      'same as the group name. The CSV fields include the name of the '
      'function, the file name and the object with the definition, and the CWP '
      'inclusive count and inclusive count fraction values. The entries are '
      'sorted in descending order based on the inclusive count value.')

  options = parser.parse_args(arguments)

  return options


def Main(argv):
  options = ParseArguments(argv)

  if os.path.exists(options.common_functions_path):
    shutil.rmtree(options.common_functions_path)

  os.makedirs(options.common_functions_path)

  if os.path.exists(options.common_functions_groups_path):
    shutil.rmtree(options.common_functions_groups_path)

  os.makedirs(options.common_functions_groups_path)

  if os.path.exists(options.extra_cwp_functions_groups_path):
    shutil.rmtree(options.extra_cwp_functions_groups_path)

  os.makedirs(options.extra_cwp_functions_groups_path)

  hot_functions_processor = HotFunctionsProcessor(
      options.pprof_top_path, options.pprof_tree_path,
      options.cwp_inclusive_count_file,
      options.cwp_pairwise_inclusive_count_file,
      options.cwp_function_groups_file, options.common_functions_path,
      options.common_functions_groups_path, options.benchmark_set_metrics_file,
      options.extra_cwp_functions_file, options.extra_cwp_functions_groups_file,
      options.extra_cwp_functions_groups_path)

  hot_functions_processor.ProcessHotFunctions()


if __name__ == '__main__':
  Main(sys.argv[1:])