Clang-Tidy: Generate protobuf of build warnings

This CL has warn.py generate a protobuffer containing information from
the build log warnings, with each individual warning as its own protobuf
message. This feature can be invoked with the --protopath command line argument
to warn.py.

BUG=NONE
TEST=Unit tests (including ones added for this functionality) pass

Change-Id: I59014674b45de04f53cf89737b3227a1815c389f
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1644060
Reviewed-by: Caroline Tice <cmtice@chromium.org>
Tested-by: Emma Vukelj <emmavukelj@google.com>

3 files changed, 152 insertions, 21 deletions

diff --git a/clang_tidy/clang_tidy_warn.py b/clang_tidy/clang_tidy_warn.py
index 011e7440..57600aa4 100755
--- a/clang_tidy/clang_tidy_warn.py
+++ b/clang_tidy/clang_tidy_warn.py
@@ -94,6 +94,7 @@ import signal
 import sys
 
 from clang_tidy_warn_patterns import warn_patterns, Severity
+import warning_pb2
 
 # TODO: move the parser code into a function
 parser = argparse.ArgumentParser(description='Convert a build log into HTML')
@@ -107,6 +108,10 @@ parser.add_argument(
     action='store_true',
     default=False)
 parser.add_argument(
+    '--protopath',
+    help='Save protobuffer warning file to the passed absolute path',
+    default=None)
+parser.add_argument(
     '--byproject',
     help='Separate warnings in HTML output by project names',
     action='store_true',
@@ -1584,6 +1589,56 @@ def dump_csv(writer):
   writer.writerow([total, '', 'All warnings'])
 
 
+def dump_protobuf(write_file):
+  """Dump warnings in protobuf format to file given"""
+  warnings = generate_protobufs()
+  protobufsString = ""
+  # write all protobufs to file (length delimited for now)
+  maxNumberLength = len(str(sys.maxint))
+  for warning in warnings:
+    warningString = warning.SerializeToString()
+    protobufsString += str(len(warningString)).zfill(maxNumberLength)
+    protobufsString += warningString
+  write_file.write(protobufsString)
+
+
+def parse_compiler_output(compiler_output):
+  # Parse compiler output for relevant info
+  split_output = compiler_output.split(":")
+  if len(split_output) < 3:
+    return  # lacks path:line_number:col_number <warning> format
+  file_path = split_output[0]
+  line_number = int(split_output[1])
+  col_number = int(split_output[2].split(" ")[0])
+  warning_message = ":".join(split_output[3:])
+  return file_path, line_number, col_number, warning_message
+
+
+def generate_protobufs():
+  """Convert warning_records to protobufs"""
+  warnings = []
+  for warning_record in warning_records:
+    warn_pattern = warn_patterns[warning_record[0]]
+    compiler_output = warning_messages[warning_record[2]]
+
+    # create warning protobuf
+    warning = warning_pb2.warning()
+    warning.severity = warn_pattern['severity']
+    warning.warning_text = warn_pattern['description']
+
+    parsed_output = parse_compiler_output(compiler_output)
+    if parsed_output is None:  # lacks structure for parsing
+      warning.matching_compiler_output = parsed_output
+    else:
+      file_path, line_number, col_number, warning_message = parsed_output
+      warning.file_path = file_path
+      warning.line_number = line_number
+      warning.col_number = col_number
+      warning.matching_compiler_output = warning_message
+    warnings.append(warning)
+  return warnings
+
+
 def main():
   warning_lines = parse_input_file(open(args.buildlog, 'r'))
   parallel_classify_warnings(warning_lines)
@@ -1598,6 +1653,10 @@ def main():
   else:
     dump_html()
 
+  if args.protopath:
+    with open(args.protopath, 'wb') as f:
+      dump_protobuf(f)
+
 
 # Run main function if warn.py is the main program.
 if __name__ == '__main__':
diff --git a/clang_tidy/clang_tidy_warn_test.py b/clang_tidy/clang_tidy_warn_test.py
index 86937bc8..d8b3866f 100755
--- a/clang_tidy/clang_tidy_warn_test.py
+++ b/clang_tidy/clang_tidy_warn_test.py
@@ -19,6 +19,8 @@ import unittest
 from contextlib import contextmanager
 import StringIO
 
+import warning_pb2
+
 
 def get_test_vars():
   """create artificial warn_patterns and project names for testing purposes"""
@@ -29,25 +31,49 @@ def get_test_vars():
       'projects': {
           'ProjectA': 2,
           'ProjectB': 0
-      }
-  }, {
-      'severity': 1,
-      'projects': {
-          'ProjectA': 1,
-          'ProjectB': 3
-      }
-  }, {
-      'severity': 2,
-      'projects': {
-          'ProjectA': 0,
-          'ProjectB': 6
-      }
-  }]
-  for s in range(9):
+      },
+      'description': 'Test warning of severity 0 (FIXMENOW)'
+  },
+                   {
+                       'severity': 1,
+                       'projects': {
+                           'ProjectA': 1,
+                           'ProjectB': 3
+                       },
+                       'description': 'Test warning of severity 1 (HIGH)'
+                   },
+                   {
+                       'severity': 2,
+                       'projects': {
+                           'ProjectA': 0,
+                           'ProjectB': 6
+                       },
+                       'description': 'Test warning of severity 2 (MEDIUM)'
+                   }]
+  for s in range(9):  # pad warn_patterns with severities we are not using
     if s >= len(warn_patterns):
       warn_patterns.append({'severity': s, 'projects': {}})
+      warn_patterns[s]['description'] = ""
     warn_patterns[s]['members'] = []
-    warn_patterns[s]['description'] = ""
+
+  warning_messages = [
+      "/ProjectB:1:1 warning: (1) Project B of severity 1",
+      "/ProjectB:1:1 warning: (2) Project B of severity 1",
+      "/ProjectA:22:23 warning: (3) Project B of severity 1",
+      "/ProjectA:22:23 (1) Project A of severity 0",
+      "/ProjectA:22:23 warning: (2) Project A of severity 0",
+      "/ProjectA:22:23 warning: Project A of severity 1",
+      "/ProjectB:1:1 warning: (1) Project B of severity 2",
+      "/ProjectB:1:1 warning: (2) Project B of severity 2",
+      "/ProjectB:1:1 warning: (3) Project B of severity 2",
+      "/ProjectB:1:1 warning: (4) Project B of severity 2",
+      "/ProjectB:1:1 warning: (5) Project B of severity 2",
+      "/ProjectB:1:1 warning: (6) Project B of severity 2"
+  ]
+  # [ warn_patterns index, project_names index, warning_messages index
+  warning_records = [[1, 1, 0], [1, 1, 1], [1, 1, 2], [0, 0, 3], [0, 0, 4],
+                     [1, 0, 5], [2, 1, 6], [2, 1, 7], [2, 1, 8], [2, 1, 9],
+                     [2, 1, 10], [2, 1, 11]]
 
   expected_warnings = {
       'ProjectA': {
@@ -102,7 +128,9 @@ def get_test_vars():
       'total_by_severity': expected_total_by_severity,
       'total_all_projects': expected_total_all_projects,
       'stats_rows': expected_stats_rows,
-      'count_severity_total': expected_count_severity_total
+      'count_severity_total': expected_count_severity_total,
+      'warning_messages': warning_messages,
+      'warning_records': warning_records,
   }
 
   return res
@@ -112,19 +140,27 @@ def put_test_vars():
   # save old warn patterns to reset to following this test
   actual_warn_patterns = ct_warn.warn_patterns
   actual_project_names = ct_warn.project_names
+  actual_warning_messages = ct_warn.warning_messages
+  actual_warning_records = ct_warn.warning_records
 
   # run test w specified inputs
   expected = get_test_vars()
 
   ct_warn.warn_patterns = expected['warn_patterns']
   ct_warn.project_names = expected['project_names']
-  return actual_warn_patterns, actual_project_names
+  ct_warn.warning_messages = expected['warning_messages']
+  ct_warn.warning_records = expected['warning_records']
+  return (actual_warn_patterns, actual_project_names, actual_warning_messages,
+          actual_warning_records)
 
 
-def remove_test_vars(actual_warn_patterns, actual_project_names):
+def remove_test_vars(actual_warn_patterns, actual_project_names,
+                     actual_warning_messages, actual_warning_records):
   # reset to actual vals
   ct_warn.project_names = actual_project_names
   ct_warn.warn_patterns = actual_warn_patterns
+  ct_warn.warning_messages = actual_warning_messages
+  ct_warn.warning_records = actual_warning_records
 
 
 def setup_classify():
@@ -140,11 +176,13 @@ def setup_classify():
 
 @contextmanager
 def test_vars():
-  actual_warn_patterns, actual_project_names = put_test_vars()
+  actual_warn_patterns, actual_project_names, actual_warning_messages, \
+    actual_warning_records = put_test_vars()
   try:
     yield
   finally:
-    remove_test_vars(actual_warn_patterns, actual_project_names)
+    remove_test_vars(actual_warn_patterns, actual_project_names,
+                     actual_warning_messages, actual_warning_records)
 
 
 class Tests(unittest.TestCase):
@@ -223,6 +261,18 @@ class Tests(unittest.TestCase):
                                                       "testing")
         self.assertEqual(count_severity_total, total[1])
 
+  def test_data_to_protobuf(self):
+    with test_vars():
+      parsed_warning_messages = []
+      for message in ct_warn.warning_messages:
+        parsed_warning_messages.append(
+            ct_warn.parse_compiler_output(message)[3])
+
+      warnings = ct_warn.generate_protobufs()
+      self.assertEqual(len(parsed_warning_messages), len(warnings))
+      for warning in warnings:  # check that each warning was found
+        self.assertIn(warning.matching_compiler_output, parsed_warning_messages)
+
 
 if __name__ == '__main__':
   unittest.main()
diff --git a/clang_tidy/warnings.proto b/clang_tidy/warnings.proto
new file mode 100644
index 00000000..75339859
--- /dev/null
+++ b/clang_tidy/warnings.proto
@@ -0,0 +1,22 @@
+syntax = "proto2";
+
+message warning {
+ enum severity_class {
+    FIXMENOW = 0;
+    HIGH = 1;
+    MEDIUM = 2;
+    LOW = 3;
+    ANALYZER = 4;
+    TIDY = 5;
+    HARMLESS = 6;
+    UNKNOWN = 7;
+    SKIP = 8;
+  }
+
+  optional severity_class severity = 1;
+  optional string warning_text = 2;
+  optional string file_path = 4;
+  optional int32 line_number = 5;
+  optional int32 col_number = 6;
+  optional string matching_compiler_output = 7;
+}