user activity: Added group extra CWP functions functionality.

Write the data of the functions that belong to a group in a CSV file,
sorted by their inclusive count value, in descendant order.

Create a CSV file containing the name of the groups, their common path,
the total inclusive count value of all the functions declared in files
that share the common path, sorted in descendant order by the inclusive
count value.

BUG=None
TEST=process_hot_functions_unittest passes

Change-Id: Ie0fe406ec0fefed02a5547dffa02e650dddd783a
Reviewed-on: https://chrome-internal-review.googlesource.com/282198
Reviewed-by: George Burgess <gbiv@google.com>
Reviewed-by: Ting-Yuan Huang <laszio@google.com>
Tested-by: Evelina Dumitrescu <evelinad@google.com>
Commit-Queue: Evelina Dumitrescu <evelinad@google.com>
Reviewed-on: https://chromium-review.googlesource.com/435904
Commit-Ready: Luis Lozano <llozano@chromium.org>
Tested-by: Luis Lozano <llozano@chromium.org>
Reviewed-by: Luis Lozano <llozano@chromium.org>

1 files changed, 153 insertions, 8 deletions

diff --git a/user_activity_benchmarks/process_hot_functions.py b/user_activity_benchmarks/process_hot_functions.py
index c6084322..3c7d65df 100755
--- a/user_activity_benchmarks/process_hot_functions.py
+++ b/user_activity_benchmarks/process_hot_functions.py
@@ -17,6 +17,10 @@ For each pprof output file, the tool will output a file that contains the hot
 functions present also in the CWP hot functions file. Afterwards, it extracts
 the functions that are present in the CWP functions file and not in the
 pprof output files.
+
+Optionally, it will organize the extra CWP functions in groups that have to
+represent a ChromeOS component. A function belongs to a group that is defined
+by a given file path if it is declared in a file that shares that path.
 """
 
 import argparse
@@ -39,7 +43,9 @@ class HotFunctionsProcessor(object):
   NOT_COMMON_FUNCTION = 0
 
   def __init__(self, pprof_path, cwp_functions_file, common_functions_path,
-               extra_cwp_functions_file):
+               extra_cwp_functions_file, cwp_function_groups_file,
+               cwp_function_groups_statistics_file,
+               cwp_function_groups_file_prefix):
     """Initializes the HotFunctionsProcessor.
 
     Args:
@@ -49,11 +55,21 @@ class HotFunctionsProcessor(object):
         pprof common functions should be stored.
       extra_cwp_functions_file: The file where should be stored the CWP
         functions that are not in the given pprof output files.
+      cwp_function_groups_file: The name of the file containing the groups of
+        functions.
+      cwp_function_groups_statistics_file: The name of the file containing the
+        statistics for the function groups.
+      cwp_function_groups_file_prefix: The prefix of the files that will store
+        the function statistics for each function group.
     """
     self._pprof_path = pprof_path
     self._cwp_functions_file = cwp_functions_file
     self._common_functions_path = common_functions_path
     self._extra_cwp_functions_file = extra_cwp_functions_file
+    self._cwp_function_groups_file = cwp_function_groups_file
+    self._cwp_function_groups_statistics_file = \
+        cwp_function_groups_statistics_file
+    self._cwp_function_groups_file_prefix = cwp_function_groups_file_prefix
 
   def ProcessHotFunctions(self):
     """Does the processing of the hot functions."""
@@ -63,6 +79,13 @@ class HotFunctionsProcessor(object):
                                    self._cwp_functions_file)
 
     self.ExtractExtraFunctions(cwp_statistics, self._extra_cwp_functions_file)
+    if all([self._cwp_function_groups_file,
+            self._cwp_function_groups_statistics_file,
+            self._cwp_function_groups_file_prefix]):
+      self.GroupExtraFunctions(cwp_statistics,
+                               self._cwp_function_groups_file_prefix,
+                               self._cwp_function_groups_file,
+                               self._cwp_function_groups_statistics_file)
 
   def ParseCWPStatistics(self, cwp_statistics_file_name):
     """Parses the contents of the file containing the CWP data.
@@ -161,25 +184,124 @@ class HotFunctionsProcessor(object):
 
     return cwp_statistics
 
+  @staticmethod
+  def ParseFunctionGroups(cwp_function_groups_lines):
+    """Parses the contents of the function groups file.
+
+    Args:
+      cwp_function_groups_lines: A list of the lines contained in the CWP
+        function groups file.
+    Returns:
+      A list of tuples containing the group name, the file path, the total
+      number of inclusive count values for that group, a list that will contain
+      the CWP statistics of the functions declared in files that share the file
+      path.
+    """
+    cwp_function_groups = []
+
+    for line in cwp_function_groups_lines:
+      group_name, file_path = line.split()
+      cwp_function_groups.append((group_name, file_path, 0, []))
+
+    return cwp_function_groups
+
+  def GroupExtraFunctions(self, cwp_statistics, cwp_function_groups_file_prefix,
+                          cwp_function_groups_file,
+                          cwp_function_groups_statistics_file):
+    """Groups the functions that are in the CWP statistics and not in the pprof
+    output. A function belongs to a group that is defined by a given file path
+    if it is declared in a file that shares that path.
+
+    Writes the data of the functions that belong to a group in a file, sorted
+    by their inclusive count value, in descendant order. The file name is
+    composed by the cwp_function_groups_file_prefix and the name of the group.
+    The file is in CSV format, containing the fields: function name, file name,
+    object name, inclusive count.
+
+    It creates a CSV file containing the name of the groups, their
+    common path, the total inclusive count value of all the functions declared
+    in files that share the common path, sorted in descendant order by the
+    inclusive count value.
+
+    Args:
+      cwp_statistics: A dict containing the CWP statistics.
+      cwp_function_groups_file_prefix: The prefix used for naming the files that
+        the function data for a specific group.
+      cwp_function_groups_file: The name of the file containing the groups of
+        functions.
+      cwp_function_groups_statistics_file: The name of the file that will
+        contain the statistics for the function groups.
+    """
+    with open(cwp_function_groups_file, 'r') as input_file:
+      cwp_function_groups = self.ParseFunctionGroups(input_file.readlines())
+
+    for function, statistics in cwp_statistics.iteritems():
+      if statistics[1] == self.COMMON_FUNCTION:
+        continue
+      file_name = function.split(',')[1]
+      group_inclusive_count = int(statistics[0].split(',')[1])
+      for i, group in enumerate(cwp_function_groups):
+        group_common_path = group[1]
+
+        # The order of the groups mentioned in the cwp_functions_groups
+        # matters. A function declared in a file will belong to the first
+        # mentioned group that matches it's path to the one of the file.
+        # It is possible to have multiple paths that belong to the same group.
+        if group_common_path in file_name:
+          group_name = group[0]
+          group_inclusive_count += group[2]
+          group_lines = group[3]
+
+          group_lines.append(','.join([function, statistics[0]]))
+          cwp_function_groups[i] = (group_name, group_common_path,
+                                    group_inclusive_count, group_lines)
+          break
+
+    group_statistics_lines = []
+
+    for group_name, group_path, group_inclusive_count, group_lines in \
+        cwp_function_groups:
+      group_statistics_lines.append(','.join([group_name, group_path,
+                                              str(group_inclusive_count)]))
+      if group_lines:
+        # Sort the output in descendant order based on the inclusive_count
+        # value.
+        group_lines.sort(key=lambda x: int(x.split(',')[-1]), reverse=True)
+        group_lines.insert(0, 'function,file,dso,inclusive_count')
+        group_file_name = cwp_function_groups_file_prefix + group_name
+
+        with open(group_file_name, 'w') as output_file:
+          output_file.write('\n'.join(group_lines))
+
+    group_statistics_lines.sort(
+        key=lambda x: int(x.split(',')[2]), reverse=True)
+    group_statistics_lines.insert(0, 'group,shared_path,inclusive_count')
+
+    with open(cwp_function_groups_statistics_file, 'w') as output_file:
+      output_file.write('\n'.join(group_statistics_lines))
+
   def ExtractExtraFunctions(self, cwp_statistics, extra_cwp_functions_file):
     """Gets the functions that are in the CWP file, but not in the pprof output.
 
     Writes the functions and their statistics in the extra_cwp_functions_file
-    file. The file is in CSV format, containing the fields: function name,
-    file name, object name, inclusive count.
+    file. The output is sorted based on the inclusive_count value. The file is
+    in CSV format, containing the fields: function name, file name, object name,
+    inclusive count.
 
     Args:
       cwp_statistics: A dict containing the CWP statistics.
       extra_cwp_functions_file: The file where should be stored the CWP
         functions and statistics that are marked as NOT_COMMON_FUNCTIONS.
     """
-    output_lines = ['function,file,dso,inclusive_count']
+    output_lines = []
 
     for function, statistics in cwp_statistics.iteritems():
       if statistics[1] == self.NOT_COMMON_FUNCTION:
-        output_lines.append(function + ',' + statistics[0])
+        output_lines.append(','.join([function, statistics[0]]))
 
     with open(extra_cwp_functions_file, 'w') as output_file:
+      output_lines.sort(key=lambda x: int(x.split(',')[-1]), reverse=True)
+      output_lines.insert(0, 'function,file,dso,inclusive_count')
       output_file.write('\n'.join(output_lines))
 
 
@@ -225,6 +347,27 @@ def ParseArguments(arguments):
       'the file names with the definition, the object '
       'file and the CWP inclusive count values, comma '
       'separated.')
+  parser.add_argument(
+      '-g',
+      '--cwp_function_groups_file',
+      dest='cwp_function_groups_file',
+      help='The file that will contain the CWP function groups.'
+      'A line consists in the group name and a file path. A group must '
+      'represent a ChromeOS component.')
+  parser.add_argument(
+      '-s',
+      '--cwp_function_groups_statistics_file',
+      dest='cwp_function_groups_statistics_file',
+      help='The file that will contain the total inclusive count values of CWP '
+      'function groups in CSV format. A line will contain the name of the '
+      'group, the common path, the total inclusive count value of all the'
+      'functions declared in files that share the common path.')
+  parser.add_argument(
+      '-x',
+      '--cwp_function_groups_file_prefix',
+      dest='cwp_function_groups_file_prefix',
+      help='The prefix of the files that will store the function statistics '
+      'for each function group.')
 
   options = parser.parse_args(arguments)
 
@@ -234,9 +377,11 @@ def ParseArguments(arguments):
 def Main(argv):
   options = ParseArguments(argv)
 
-  hot_functions_processor = HotFunctionsProcessor(options.pprof_path, \
-    options.cwp_hot_functions_file, options.common_functions_path, \
-    options.extra_cwp_functions_file)
+  hot_functions_processor = HotFunctionsProcessor(options.pprof_path,
+    options.cwp_hot_functions_file, options.common_functions_path,
+    options.extra_cwp_functions_file, options.cwp_function_groups_file,
+    options.cwp_function_groups_statistics_file,
+    options.cwp_function_groups_file_prefix)
 
   hot_functions_processor.ProcessHotFunctions()