toolchain-utils: Add a script to do post-process on Chrome orderfile.

This script takes an orderfile that contains symbols ordered by C3,
and an nm result of the Chrome binary and perform post-processing,
including add markers to begin and end, and insert all the builtin
functions in the Chrome binary.

The script is used by chromite when generating orderfile.

The CL is intended to be landed at the same time of crrev.com/c/1583077.
BUG=chromium:950627
TEST=Unittest and tryjob together with crrev.com/c/1583077

Change-Id: Ib8e85efdcf0dca053febd65c4e1c8350bf410437
Reviewed-on: https://chromium-review.googlesource.com/1605142
Tested-by: Tiancong Wang <tcwang@google.com>
Commit-Ready: ChromeOS CL Exonerator Bot <chromiumos-cl-exonerator@appspot.gserviceaccount.com>
Legacy-Commit-Queue: Commit Bot <commit-bot@chromium.org>
Reviewed-by: George Burgess <gbiv@chromium.org>

2 files changed, 185 insertions, 0 deletions

diff --git a/orderfile/post_process_orderfile.py b/orderfile/post_process_orderfile.py
new file mode 100755
index 00000000..e24ab1cd
--- /dev/null
+++ b/orderfile/post_process_orderfile.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Script to do post-process on orderfile generated by C3.
+
+The goal of this script is to take in an orderfile generated by C3, and do
+the following post process:
+
+1. Take in the result of nm command on Chrome binary to find out all the
+Builtin functions and put them after the input symbols.
+
+2. Put special markers "chrome_begin_ordered_code" and "chrome_end_ordered_code"
+in begin and end of the file.
+
+The results of the file is intended to be uploaded and consumed when linking
+Chrome in Chrome OS.
+"""
+
+from __future__ import division, print_function
+
+import argparse
+import os
+import sys
+
+
+def _parse_nm_output(stream):
+  for line in (line.rstrip() for line in stream):
+    if not line:
+      continue
+
+    pieces = line.split()
+    if len(pieces) != 3:
+      continue
+
+    _, ty, symbol = pieces
+    if ty not in 'tT':
+      continue
+
+    # We'll sometimes see synthesized symbols that start with $. There isn't
+    # much we can do about or with them, regrettably.
+    if symbol.startswith('$'):
+      continue
+
+    yield symbol
+
+
+def _remove_duplicates(iterable):
+  seen = set()
+  for item in iterable:
+    if item in seen:
+      continue
+    seen.add(item)
+    yield item
+
+
+def run(c3_ordered_stream, chrome_nm_stream, output_stream):
+  head_marker = 'chrome_begin_ordered_code'
+  tail_marker = 'chrome_end_ordered_code'
+
+  c3_ordered_syms = [x.strip() for x in c3_ordered_stream.readlines()]
+  all_chrome_syms = set(_parse_nm_output(chrome_nm_stream))
+  # Sort by name, so it's predictable. Otherwise, these should all land in the
+  # same hugepage anyway, so order doesn't matter as much.
+  builtin_syms = sorted(s for s in all_chrome_syms if s.startswith('Builtins_'))
+  output = _remove_duplicates([head_marker] + c3_ordered_syms + builtin_syms +
+                              [tail_marker])
+  output_stream.write('\n'.join(output))
+
+
+def main(argv):
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--chrome_nm', required=True, dest='chrome_nm')
+  parser.add_argument('--input', required=True, dest='input_file')
+  parser.add_argument('--output', required=True, dest='output_file')
+
+  options = parser.parse_args(argv)
+
+  if not os.path.exists(options.input_file):
+    sys.exit('Input orderfile doesn\'t exist.')
+
+  with open(options.input_file) as in_stream, \
+  open(options.chrome_nm) as chrome_nm_stream, \
+  open(options.output_file, 'w') as out_stream:
+    run(in_stream, chrome_nm_stream, out_stream)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))
diff --git a/orderfile/post_process_orderfile_test.py b/orderfile/post_process_orderfile_test.py
new file mode 100755
index 00000000..2532b8b3
--- /dev/null
+++ b/orderfile/post_process_orderfile_test.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tests for post_process_orderfile.py."""
+
+from __future__ import division, print_function
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import post_process_orderfile
+
+
+def _write_nm_file(name):
+  with open(name, 'w') as out:
+    out.write('000001 s NotAValidSymbol1\n')
+    out.write('000002 S NotAValidSymbol2\n')
+    out.write('000010 t FirstValidSymbol\n')
+    out.write('000012 t \n')
+    out.write('000020 T Builtins_SecondValidSymbol\n')
+    out.write('000030 T $SymbolToIgnore\n')
+    out.write('000036 T Builtins_LastValidSymbol\n')
+
+
+def _write_orderfile(name):
+  with open(name, 'w') as out:
+    out.write('SymbolOrdered1\n')
+    out.write('SymbolOrdered2\n')
+
+
+def _cleanup(files):
+  for f in files:
+    shutil.rmtree(f, ignore_errors=True)
+
+
+class Tests(unittest.TestCase):
+  """All of our tests for post_process_orderfile."""
+
+  #pylint: disable=protected-access
+  def test__parse_nm_output(self):
+    temp_dir = tempfile.mkdtemp()
+    self.addCleanup(_cleanup, [temp_dir])
+    chrome_nm_file = os.path.join(temp_dir, 'chrome_nm.txt')
+    _write_nm_file(chrome_nm_file)
+    with open(chrome_nm_file) as f:
+      results = list(post_process_orderfile._parse_nm_output(f))
+      self.assertEqual(len(results), 3)
+      self.assertIn('FirstValidSymbol', results)
+      self.assertIn('Builtins_SecondValidSymbol', results)
+      self.assertIn('Builtins_LastValidSymbol', results)
+
+  def test__remove_duplicates(self):
+    duplicates = ['marker1', 'marker2', 'marker3', 'marker2', 'marker1']
+    results = list(post_process_orderfile._remove_duplicates(duplicates))
+    self.assertEqual(results, ['marker1', 'marker2', 'marker3'])
+
+  def test_run(self):
+    temp_dir = tempfile.mkdtemp()
+    self.addCleanup(_cleanup, [temp_dir])
+    orderfile_input = os.path.join(temp_dir, 'orderfile.in.txt')
+    orderfile_output = os.path.join(temp_dir, 'orderfile.out.txt')
+    chrome_nm_file = os.path.join(temp_dir, 'chrome_nm.txt')
+    _write_nm_file(chrome_nm_file)
+    _write_orderfile(orderfile_input)
+    with open(orderfile_input) as in_stream, \
+    open(orderfile_output, 'w') as out_stream, \
+    open(chrome_nm_file) as chrome_nm_stream:
+      post_process_orderfile.run(in_stream, chrome_nm_stream, out_stream)
+
+    with open(orderfile_output) as check:
+      results = [x.strip() for x in check.readlines()]
+      self.assertEqual(
+          results,
+          [
+              # Start marker should be put first.
+              'chrome_begin_ordered_code',
+              # Symbols in orderfile come next.
+              'SymbolOrdered1',
+              'SymbolOrdered2',
+              # Builtin functions in chrome_nm come next, and sorted.
+              'Builtins_LastValidSymbol',
+              'Builtins_SecondValidSymbol',
+              # Last symbol should be end marker.
+              'chrome_end_ordered_code'
+          ])
+
+
+if __name__ == '__main__':
+  unittest.main()