Support for custom NM/readelf binaries in your toolchain.

Background: nm and readelf are used by ninja.py to generate
.TOC files. These .TOC files are what determine when something
needs rebuilding.

ninja.py uses distinct commands for host vs target tools for most things
in the toolchain (cc, c++, ld,), via make_global_settings, but it fails to do so
for nm & readelf. This is not by design, but it has worked thus far by chance:
The default 'nm' and 'readelf' in most people's PATH are the system variants,
and the Linux versions of these tools happen to work.

However, the project I'm working on has engineers on Macs developing for
android. The system-supplied 'nm' and 'readelf' on Mac do NOT work for
e.g. Android arm binaries, which leads me to this fix. This fix allows for
specifying NM and READELF via make_global_settings, so we can point those
variables to the correct ones for the given toolchain/target.

BUG=
R=thakis@chromium.org

Review URL: https://codereview.chromium.org/164023009

git-svn-id: http://gyp.googlecode.com/svn/trunk@1971 78cadc50-ecff-11dd-a971-7dbc132099af

13 files changed, 229 insertions, 11 deletions

diff --git a/pylib/gyp/generator/ninja.py b/pylib/gyp/generator/ninja.py
index 4eafb71c..3336a896 100644
--- a/pylib/gyp/generator/ninja.py
+++ b/pylib/gyp/generator/ninja.py
@@ -805,6 +805,8 @@ class NinjaWriter:
       self.ninja.variable('cxx', '$cxx_host')
       self.ninja.variable('ld', '$ld_host')
       self.ninja.variable('ldxx', '$ldxx_host')
+      self.ninja.variable('nm', '$nm_host')
+      self.ninja.variable('readelf', '$readelf_host')
 
     if self.flavor != 'mac' or len(self.archs) == 1:
       return self.WriteSourcesForArch(
@@ -1742,6 +1744,10 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
   cc_host_global_setting = None
   cxx_host_global_setting = None
   clang_cl = None
+  nm = 'nm'
+  nm_host = 'nm'
+  readelf = 'readelf'
+  readelf_host = 'readelf'
 
   build_file, _, _ = gyp.common.ParseQualifiedTarget(target_list[0])
   make_global_settings = data[build_file].get('make_global_settings', [])
@@ -1769,6 +1775,14 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
       ld = os.path.join(build_to_root, value)
     if key == 'LD.host':
       ld_host = os.path.join(build_to_root, value)
+    if key == 'NM':
+      nm = os.path.join(build_to_root, value)
+    if key == 'NM.host':
+      nm_host = os.path.join(build_to_root, value)
+    if key == 'READELF':
+      readelf = os.path.join(build_to_root, value)
+    if key == 'READELF.host':
+      readelf_host = os.path.join(build_to_root, value)
     if key.endswith('_wrapper'):
       wrappers[key[:-len('_wrapper')]] = os.path.join(build_to_root, value)
 
@@ -1816,6 +1830,13 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
     master_ninja.variable('ld', CommandWithWrapper('LINK', wrappers, ld))
     master_ninja.variable('ldxx', CommandWithWrapper('LINK', wrappers, ldxx))
     master_ninja.variable('ar', GetEnvironFallback(['AR_target', 'AR'], ar))
+    if flavor != 'mac':
+      # Mac does not use readelf/nm for .TOC generation, so avoiding polluting
+      # the master ninja with extra unused variables.
+      master_ninja.variable(
+          'nm', GetEnvironFallback(['NM_target', 'NM'], nm))
+      master_ninja.variable(
+          'readelf', GetEnvironFallback(['READELF_target', 'READELF'], readelf))
 
   if generator_supports_multiple_toolsets:
     if not cc_host:
@@ -1824,6 +1845,9 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
       cxx_host = cxx
 
     master_ninja.variable('ar_host', GetEnvironFallback(['AR_host'], ar_host))
+    master_ninja.variable('nm_host', GetEnvironFallback(['NM_host'], nm_host))
+    master_ninja.variable('readelf_host',
+                          GetEnvironFallback(['READELF_host'], readelf_host))
     cc_host = GetEnvironFallback(['CC_host'], cc_host)
     cxx_host = GetEnvironFallback(['CXX_host'], cxx_host)
 
@@ -1945,8 +1969,8 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params,
         % { 'solink':
               '$ld -shared $ldflags -o $lib -Wl,-soname=$soname %(suffix)s',
             'extract_toc':
-              ('{ readelf -d $lib | grep SONAME ; '
-               'nm -gD -f p $lib | cut -f1-2 -d\' \'; }')})
+              ('{ $readelf -d $lib | grep SONAME ; '
+               '$nm -gD -f p $lib | cut -f1-2 -d\' \'; }')})
 
     master_ninja.rule(
       'solink',
diff --git a/test/compiler-override/compiler.gyp b/test/compiler-override/compiler-exe.gyp
index c2f3002f..c2f3002f 100644
--- a/test/compiler-override/compiler.gyp
+++ b/test/compiler-override/compiler-exe.gyp
diff --git a/test/compiler-override/compiler-shared-lib.gyp b/test/compiler-override/compiler-shared-lib.gyp
new file mode 100644
index 00000000..d3e43161
--- /dev/null
+++ b/test/compiler-override/compiler-shared-lib.gyp
@@ -0,0 +1,16 @@
+# Copyright (c) 2012 Google Inc. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+{
+  'targets': [
+    {
+      'target_name': 'hello-lib',
+      'type': 'shared_library',
+      'sources': [
+        'test.c',
+        'cxxtest.cc',
+      ],
+    },
+  ],
+}
diff --git a/test/compiler-override/gyptest-compiler-env-toolchain.py b/test/compiler-override/gyptest-compiler-env-toolchain.py
new file mode 100644
index 00000000..c27be242
--- /dev/null
+++ b/test/compiler-override/gyptest-compiler-env-toolchain.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 Google Inc. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""
+Verifies that the user can override the compiler and linker using
+CC/CXX/NM/READELF environment variables.
+"""
+
+import TestGyp
+import os
+import copy
+import sys
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+if sys.platform == 'win32':
+  # cross compiling not supported by ninja on windows
+  # and make not supported on windows at all.
+  sys.exit(0)
+
+# Clear any existing compiler related env vars.
+for key in ['CC', 'CXX', 'LINK', 'CC_host', 'CXX_host', 'LINK_host',
+            'NM_target', 'READELF_target']:
+  if key in os.environ:
+    del os.environ[key]
+
+
+def CheckCompiler(test, gypfile, check_for, run_gyp):
+  if run_gyp:
+    test.run_gyp(gypfile)
+  test.build(gypfile)
+
+  test.must_contain_all_lines(test.stdout(), check_for)
+
+
+test = TestGyp.TestGyp(formats=['ninja'])
+# Must set the test format to something with a flavor (the part after the '-')
+# in order to test the desired behavior. Since we want to run a non-host
+# toolchain, we have to set the flavor to something that the ninja generator
+# doesn't know about, so it doesn't default to the host-specific tools (e.g.,
+# 'otool' on mac to generate the .TOC).
+#
+# Note that we can't just pass format=['ninja-some_toolchain'] to the
+# constructor above, because then this test wouldn't be recognized as a ninja
+# format test.
+test.formats = ['ninja-some_flavor']
+
+
+def TestTargetOverideSharedLib():
+  # The std output from nm and readelf is redirected to files, so we can't
+  # expect their output to appear. Instead, check for the files they create to
+  # see if they actually ran.
+  expected = ['my_cc.py', 'my_cxx.py', 'FOO']
+
+  # Check that CC, CXX, NM, READELF, set target compiler
+  env = {'CC': 'python %s/my_cc.py FOO' % here,
+         'CXX': 'python %s/my_cxx.py FOO' % here,
+         'NM': 'python %s/my_nm.py' % here,
+         'READELF': 'python %s/my_readelf.py' % here}
+
+  with TestGyp.LocalEnv(env):
+    CheckCompiler(test, 'compiler-shared-lib.gyp', expected, True)
+    test.must_contain(test.built_file_path('RAN_MY_NM'), 'RAN_MY_NM')
+    test.must_contain(test.built_file_path('RAN_MY_READELF'), 'RAN_MY_READELF')
+    test.unlink(test.built_file_path('RAN_MY_NM'))
+    test.unlink(test.built_file_path('RAN_MY_READELF'))
+
+  # Run the same tests once the eviron has been restored.  The generated
+  # projects should have embedded all the settings in the project files so the
+  # results should be the same.
+  CheckCompiler(test, 'compiler-shared-lib.gyp', expected, False)
+  test.must_contain(test.built_file_path('RAN_MY_NM'), 'RAN_MY_NM')
+  test.must_contain(test.built_file_path('RAN_MY_READELF'), 'RAN_MY_READELF')
+
+
+TestTargetOverideSharedLib()
+test.pass_test()
diff --git a/test/compiler-override/gyptest-compiler-env.py b/test/compiler-override/gyptest-compiler-env.py
index d13d692d..6408ae11 100755
--- a/test/compiler-override/gyptest-compiler-env.py
+++ b/test/compiler-override/gyptest-compiler-env.py
@@ -15,7 +15,7 @@ import sys
 here = os.path.dirname(os.path.abspath(__file__))
 
 if sys.platform == 'win32':
-  # cross compiling not support by ninja on windows
+  # cross compiling not supported by ninja on windows
   # and make not supported on windows at all.
   sys.exit(0)
 
@@ -30,8 +30,6 @@ def CheckCompiler(test, gypfile, check_for, run_gyp):
     test.run_gyp(gypfile)
   test.build(gypfile)
 
-  # We can't test to presence of my_ld.py in the output since
-  # ninja will use CXX_target as the linker regardless
   test.must_contain_all_lines(test.stdout(), check_for)
 
 
@@ -49,8 +47,7 @@ def TestTargetOveride():
     os.environ['CXX'] = 'python %s/my_cxx.py FOO' % here
     os.environ['LINK'] = 'python %s/my_ld.py FOO_LINK' % here
 
-    CheckCompiler(test, 'compiler.gyp', expected,
-                  True)
+    CheckCompiler(test, 'compiler-exe.gyp', expected, True)
   finally:
     os.environ.clear()
     os.environ.update(oldenv)
@@ -58,8 +55,8 @@ def TestTargetOveride():
   # Run the same tests once the eviron has been restored.  The
   # generated should have embedded all the settings in the
   # project files so the results should be the same.
-  CheckCompiler(test, 'compiler.gyp', expected,
-                False)
+  CheckCompiler(test, 'compiler-exe.gyp', expected, False)
+
 
 def TestTargetOverideCompilerOnly():
   # Same test again but with that CC, CXX and not LD
@@ -68,7 +65,7 @@ def TestTargetOverideCompilerOnly():
     os.environ['CC'] = 'python %s/my_cc.py FOO' % here
     os.environ['CXX'] = 'python %s/my_cxx.py FOO' % here
 
-    CheckCompiler(test, 'compiler.gyp',
+    CheckCompiler(test, 'compiler-exe.gyp',
                   ['my_cc.py', 'my_cxx.py', 'FOO'],
                   True)
   finally:
@@ -78,7 +75,7 @@ def TestTargetOverideCompilerOnly():
   # Run the same tests once the eviron has been restored.  The
   # generated should have embedded all the settings in the
   # project files so the results should be the same.
-  CheckCompiler(test, 'compiler.gyp',
+  CheckCompiler(test, 'compiler-exe.gyp',
                 ['my_cc.py', 'my_cxx.py', 'FOO'],
                 False)
 
diff --git a/test/compiler-override/my_nm.py b/test/compiler-override/my_nm.py
new file mode 100755
index 00000000..f0f1efcf
--- /dev/null
+++ b/test/compiler-override/my_nm.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+# Copyright (c) 2014 Google Inc. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import sys
+print sys.argv
+with open('RAN_MY_NM', 'w') as f:
+  f.write('RAN_MY_NM')
diff --git a/test/compiler-override/my_readelf.py b/test/compiler-override/my_readelf.py
new file mode 100755
index 00000000..40e303cd
--- /dev/null
+++ b/test/compiler-override/my_readelf.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+# Copyright (c) 2014 Google Inc. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import sys
+print sys.argv
+with open('RAN_MY_READELF', 'w') as f:
+  f.write('RAN_MY_READELF')
diff --git a/test/make_global_settings/full-toolchain/bar.cc b/test/make_global_settings/full-toolchain/bar.cc
new file mode 100644
index 00000000..afb422ba
--- /dev/null
+++ b/test/make_global_settings/full-toolchain/bar.cc
@@ -0,0 +1 @@
+#error Not a real source file
diff --git a/test/make_global_settings/full-toolchain/foo.c b/test/make_global_settings/full-toolchain/foo.c
new file mode 100644
index 00000000..afb422ba
--- /dev/null
+++ b/test/make_global_settings/full-toolchain/foo.c
@@ -0,0 +1 @@
+#error Not a real source file
diff --git a/test/make_global_settings/full-toolchain/gyptest-make_global_settings.py b/test/make_global_settings/full-toolchain/gyptest-make_global_settings.py
new file mode 100644
index 00000000..0dd148c2
--- /dev/null
+++ b/test/make_global_settings/full-toolchain/gyptest-make_global_settings.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2014 Google Inc. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Verifies make_global_settings works with the full toolchain.
+"""
+
+import os
+import sys
+import TestGyp
+
+if sys.platform == 'win32':
+  # cross compiling not supported by ninja on windows
+  # and make not supported on windows at all.
+  sys.exit(0)
+
+test = TestGyp.TestGyp(formats=['ninja'])
+# Must set the test format to something with a flavor (the part after the '-')
+# in order to test the desired behavior. Since we want to run a non-host
+# toolchain, we have to set the flavor to something that the ninja generator
+# doesn't know about, so it doesn't default to the host-specific tools (e.g.,
+# 'otool' on mac to generate the .TOC).
+#
+# Note that we can't just pass format=['ninja-some_toolchain'] to the
+# constructor above, because then this test wouldn't be recognized as a ninja
+# format test.
+test.formats = ['ninja-some_flavor']
+
+gyp_file = 'make_global_settings.gyp'
+
+test.run_gyp(gyp_file,
+             # Teach the .gyp file about the location of my_nm.py and
+             # my_readelf.py, and the python executable.
+             '-Dworkdir=%s' % test.workdir,
+             '-Dpython=%s' % sys.executable)
+test.build(gyp_file, arguments=['-v'])
+
+expected = ['MY_CC', 'MY_CXX']
+test.must_contain_all_lines(test.stdout(), expected)
+
+test.must_contain(test.built_file_path('RAN_MY_NM'), 'RAN_MY_NM')
+test.must_contain(test.built_file_path('RAN_MY_READELF'), 'RAN_MY_READELF')
+
+test.pass_test()
diff --git a/test/make_global_settings/full-toolchain/make_global_settings.gyp b/test/make_global_settings/full-toolchain/make_global_settings.gyp
new file mode 100644
index 00000000..2c326632
--- /dev/null
+++ b/test/make_global_settings/full-toolchain/make_global_settings.gyp
@@ -0,0 +1,22 @@
+# Copyright (c) 2013 Google Inc. All rights reserved.
+# Use of this source code is governed by a BSD-style licence that can be
+# found in the LICENSE file.
+
+{
+  'make_global_settings': [
+    ['CC', '/bin/echo MY_CC'],
+    ['CXX', '/bin/echo MY_CXX'],
+    ['NM', '<(python) <(workdir)/my_nm.py'],
+    ['READELF', '<(python) <(workdir)/my_readelf.py'],
+  ],
+  'targets': [
+    {
+      'target_name': 'test',
+      'type': 'shared_library',
+      'sources': [
+        'foo.c',
+        'bar.cc',
+      ],
+    },
+  ],
+}
diff --git a/test/make_global_settings/full-toolchain/my_nm.py b/test/make_global_settings/full-toolchain/my_nm.py
new file mode 100755
index 00000000..f0f1efcf
--- /dev/null
+++ b/test/make_global_settings/full-toolchain/my_nm.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+# Copyright (c) 2014 Google Inc. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import sys
+print sys.argv
+with open('RAN_MY_NM', 'w') as f:
+  f.write('RAN_MY_NM')
diff --git a/test/make_global_settings/full-toolchain/my_readelf.py b/test/make_global_settings/full-toolchain/my_readelf.py
new file mode 100755
index 00000000..40e303cd
--- /dev/null
+++ b/test/make_global_settings/full-toolchain/my_readelf.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+# Copyright (c) 2014 Google Inc. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import sys
+print sys.argv
+with open('RAN_MY_READELF', 'w') as f:
+  f.write('RAN_MY_READELF')