diff options
author | sakal <sakal@webrtc.org> | 2017-09-05 00:16:15 -0700 |
---|---|---|
committer | Commit Bot <commit-bot@chromium.org> | 2017-09-05 07:16:15 +0000 |
commit | 67e414ce709446a3d6538bd9a4d027b698afa031 (patch) | |
tree | 45d8652b6479893cb1c4e20cc1bf9b187063d100 /tools_webrtc | |
parent | 31377a2a103f040c3c4f0b1947d69657233c21a2 (diff) | |
download | webrtc-67e414ce709446a3d6538bd9a4d027b698afa031.tar.gz |
License generation script for build_aar.py.
The script is forked from: tools_webrtc/ios/generate_licenses.py
BUG=webrtc:8182
Review-Url: https://codereview.webrtc.org/3011613002
Cr-Commit-Position: refs/heads/master@{#19679}
Diffstat (limited to 'tools_webrtc')
-rwxr-xr-x | tools_webrtc/android/build_aar.py | 16 | ||||
-rwxr-xr-x | tools_webrtc/ios/build_ios_libs.py | 11 | ||||
-rwxr-xr-x | tools_webrtc/ios/generate_licenses.py | 132 | ||||
-rw-r--r-- | tools_webrtc/libs/__init__.py | 0 | ||||
-rwxr-xr-x | tools_webrtc/libs/generate_licenses.py | 153 | ||||
-rwxr-xr-x | tools_webrtc/libs/generate_licenses_test.py | 62 |
6 files changed, 237 insertions, 137 deletions
diff --git a/tools_webrtc/android/build_aar.py b/tools_webrtc/android/build_aar.py index 54272e979e..50f1ff4e7c 100755 --- a/tools_webrtc/android/build_aar.py +++ b/tools_webrtc/android/build_aar.py @@ -34,6 +34,7 @@ import tempfile import zipfile +SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0])) DEFAULT_ARCHS = ['armeabi-v7a', 'arm64-v8a', 'x86', 'x86_64'] NEEDED_SO_FILES = ['libjingle_peerconnection_so.so'] JAR_FILE = 'lib.java/webrtc/sdk/android/libwebrtc.jar' @@ -43,6 +44,9 @@ TARGETS = [ 'webrtc/sdk/android:libjingle_peerconnection_so', ] +sys.path.append(os.path.join(SCRIPT_DIR, '..', 'libs')) +from generate_licenses import LicenseBuilder + def _ParseArgs(): parser = argparse.ArgumentParser(description='libwebrtc.aar generator.') @@ -122,6 +126,7 @@ def Build(tmp_dir, arch, use_goma, extra_gn_args): 'target_os': 'android', 'is_debug': False, 'is_component_build': False, + 'rtc_include_tests': False, 'target_cpu': _GetTargetCpu(arch), 'use_goma': use_goma } @@ -133,7 +138,7 @@ def Build(tmp_dir, arch, use_goma, extra_gn_args): _RunGN(['gen', output_directory, gn_args_str]) - ninja_args = TARGETS + ninja_args = TARGETS[:] if use_goma: ninja_args.extend(['-j', '200']) _RunNinja(output_directory, ninja_args) @@ -158,6 +163,12 @@ def Collect(aar_file, tmp_dir, arch): os.path.join(abi_dir, so_file)) +def GenerateLicenses(output_dir, tmp_dir, archs): + builder = LicenseBuilder( + [_GetOutputDirectory(tmp_dir, arch) for arch in archs], TARGETS) + builder.GenerateLicenseText(output_dir) + + def main(): args = _ParseArgs() logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) @@ -173,6 +184,9 @@ def main(): for arch in args.arch: Collect(aar_file, tmp_dir, arch) + license_dir = os.path.dirname(os.path.realpath(args.output)) + GenerateLicenses(license_dir, tmp_dir, args.arch) + shutil.rmtree(tmp_dir, True) diff --git a/tools_webrtc/ios/build_ios_libs.py b/tools_webrtc/ios/build_ios_libs.py index 98a2407da2..d78384ae76 100755 --- a/tools_webrtc/ios/build_ios_libs.py +++ b/tools_webrtc/ios/build_ios_libs.py @@ -34,6 +34,9 @@ DEFAULT_ARCHS = ENABLED_ARCHS = ['arm64', 'arm', 'x64', 'x86'] IOS_DEPLOYMENT_TARGET = '8.0' LIBVPX_BUILD_VP9 = False +sys.path.append(os.path.join(SCRIPT_DIR, '..', 'libs')) +from generate_licenses import LicenseBuilder + def _ParseArgs(): parser = argparse.ArgumentParser(description=__doc__) @@ -224,13 +227,13 @@ def main(): _RunCommand(cmd) # Generate the license file. - license_script_path = os.path.join(SCRIPT_DIR, 'generate_licenses.py') ninja_dirs = [os.path.join(args.output_dir, arch + '_libs') for arch in architectures] gn_target_full_name = '//webrtc/sdk:' + gn_target_name - cmd = [sys.executable, license_script_path, gn_target_full_name, - os.path.join(args.output_dir, SDK_FRAMEWORK_NAME)] + ninja_dirs - _RunCommand(cmd) + builder = LicenseBuilder(ninja_dirs, [gn_target_full_name]) + builder.GenerateLicenseText( + os.path.join(args.output_dir, SDK_FRAMEWORK_NAME)) + # Modify the version number. # Format should be <Branch cut MXX>.<Hotfix #>.<Rev #>. diff --git a/tools_webrtc/ios/generate_licenses.py b/tools_webrtc/ios/generate_licenses.py deleted file mode 100755 index b91314783f..0000000000 --- a/tools_webrtc/ios/generate_licenses.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/python - -# Copyright 2016 The WebRTC project authors. All Rights Reserved. -# -# Use of this source code is governed by a BSD-style license -# that can be found in the LICENSE file in the root of the source -# tree. An additional intellectual property rights grant can be found -# in the file PATENTS. All contributing project authors may -# be found in the AUTHORS file in the root of the source tree. - -"""Generates license HTML for a prebuilt version of WebRTC for iOS.""" - -import sys - -import argparse -import cgi -import os -import re -import textwrap -import subprocess - - -LIB_TO_LICENSES_DICT = { - 'boringssl': ['third_party/boringssl/src/LICENSE'], - 'expat': ['third_party/expat/files/COPYING'], - 'jsoncpp': ['third_party/jsoncpp/LICENSE'], - 'opus': ['third_party/opus/src/COPYING'], - 'protobuf': ['third_party/protobuf/LICENSE'], - 'libsrtp': ['third_party/libsrtp/LICENSE'], - 'usrsctp': ['third_party/usrsctp/LICENSE'], - 'webrtc': ['webrtc/LICENSE', 'webrtc/LICENSE_THIRD_PARTY'], - 'libvpx': ['third_party/libvpx/source/libvpx/LICENSE'], - 'libyuv': ['third_party/libyuv/LICENSE'], -} - -SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0])) -CHECKOUT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir)) -WEBRTC_ROOT = os.path.join(CHECKOUT_ROOT, 'webrtc') - - -def GetThirdPartyLibraries(buildfile_dir, target_name): - def ExtractLibName(string_list): - # Sample input: - # [" //third_party/usrsctp:usrsctp", " //webrtc:webrtc_common"] - # Sample output: - # ["usrsctp"] - return re.sub(r'\(.*\)', '', string_list).strip().split( - os.path.sep)[-1].split(':')[0] - output = subprocess.check_output( - ["gn", "desc", buildfile_dir, target_name, '--all']) .split(os.linesep) - return [ExtractLibName(x) for x in output if re.search(r'third_party', x)] - - -class LicenseBuilder(object): - - def __init__(self, buildfile_dirs, target_name): - self.buildfile_dirs = buildfile_dirs - self.target_name = target_name - - def GenerateLicenseText(self, output_dir): - # Get a list of third_party libs from gn. For fat libraries we must consider - # all architectures, hence the multiple buildfile directories. - # The `sum` function flattens the 2d list. - third_party_libs = sum([GetThirdPartyLibraries(buildfile, self.target_name) - for buildfile in self.buildfile_dirs], []) - assert len(third_party_libs) > 0 - - # Generate amalgamated list of libraries. Will exit with error if a - # lib is unrecognized. - license_libs = set() - for static_lib in third_party_libs: - license_path = LIB_TO_LICENSES_DICT.get(static_lib) - if static_lib == 'yasm': - # yasm is a build-time dep only, and doesn't need a license. - continue - if license_path is None: - print 'Missing license path for lib: %s' % static_lib - return 1 - license_libs.add(static_lib) - - # Put webrtc at the front of the list. - license_libs = sorted(license_libs) - license_libs.insert(0, 'webrtc') - - # Generate HTML. - output_license_file = open(os.path.join(output_dir, 'LICENSE.html'), 'w+') - output_license_file.write('<!DOCTYPE html>\n') - output_license_file.write('<html>\n<head>\n') - output_license_file.write('<meta charset="UTF-8">\n') - output_license_file.write('<title>Licenses</title>\n') - style_tag = textwrap.dedent('''\ - <style> - body { margin: 0; font-family: sans-serif; } - pre { background-color: #eeeeee; padding: 1em; white-space: pre-wrap; } - p { margin: 1em; white-space: nowrap; } - </style> - ''') - output_license_file.write(style_tag) - output_license_file.write('</head>\n') - - for license_lib in license_libs: - output_license_file.write('<p>%s<br/></p>\n' % license_lib) - output_license_file.write('<pre>\n') - for path in LIB_TO_LICENSES_DICT[license_lib]: - license_path = os.path.join(CHECKOUT_ROOT, path) - with open(license_path, 'r') as license_file: - license_text = cgi.escape(license_file.read(), quote=True) - output_license_file.write(license_text) - output_license_file.write('\n') - output_license_file.write('</pre>\n') - - output_license_file.write('</body>\n') - output_license_file.write('</html>') - output_license_file.close() - return 0 - - -def main(): - parser = argparse.ArgumentParser(description='Generate WebRTC LICENSE.html') - parser.add_argument('target_name', - help='Name of the GN target to generate a license for') - parser.add_argument('output_dir', - help='Directory to output LICENSE.html to.') - parser.add_argument('buildfile_dirs', nargs="+", - help='Directories containing gn generated ninja files') - args = parser.parse_args() - builder = LicenseBuilder(args.buildfile_dirs, args.target_name) - sys.exit(builder.GenerateLicenseText(args.output_dir)) - - -if __name__ == '__main__': - main() diff --git a/tools_webrtc/libs/__init__.py b/tools_webrtc/libs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools_webrtc/libs/__init__.py diff --git a/tools_webrtc/libs/generate_licenses.py b/tools_webrtc/libs/generate_licenses.py new file mode 100755 index 0000000000..b9ca02fea0 --- /dev/null +++ b/tools_webrtc/libs/generate_licenses.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python + +# Copyright 2016 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Generates license markdown for a prebuilt version of WebRTC.""" + +import sys + +import argparse +import cgi +import json +import logging +import os +import re +import subprocess + + +LIB_TO_LICENSES_DICT = { + 'android_tools': ['third_party/android_tools/LICENSE'], + 'boringssl': ['third_party/boringssl/src/LICENSE'], + 'expat': ['third_party/expat/files/COPYING'], + 'ijar': ['third_party/ijar/LICENSE'], + 'jsoncpp': ['third_party/jsoncpp/LICENSE'], + 'libc++': ['buildtools/third_party/libc++/trunk/LICENSE.TXT'], + 'libc++abi': ['buildtools/third_party/libc++abi/trunk/LICENSE.TXT'], + 'libevent': ['base/third_party/libevent/LICENSE'], + 'libjpeg_turbo': ['third_party/libjpeg_turbo/LICENSE.md'], + 'libsrtp': ['third_party/libsrtp/LICENSE'], + 'libvpx': ['third_party/libvpx/source/libvpx/LICENSE'], + 'libyuv': ['third_party/libyuv/LICENSE'], + 'openmax_dl': ['third_party/openmax_dl/LICENSE'], + 'opus': ['third_party/opus/src/COPYING'], + 'protobuf': ['third_party/protobuf/LICENSE'], + 'usrsctp': ['third_party/usrsctp/LICENSE'], + 'webrtc': ['webrtc/LICENSE', 'webrtc/LICENSE_THIRD_PARTY'], + + # Compile time dependencies, no license needed: + 'yasm': [], +} + +SCRIPT_DIR = os.path.dirname(os.path.realpath(sys.argv[0])) +CHECKOUT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir)) +WEBRTC_ROOT = os.path.join(CHECKOUT_ROOT, 'webrtc') +THIRD_PARTY_LIB_REGEX = r'^.*/third_party/([\w+]+).*$' + +class LicenseBuilder(object): + + def __init__(self, buildfile_dirs, targets): + self.buildfile_dirs = buildfile_dirs + self.targets = targets + + @staticmethod + def _ParseLibrary(dep): + """ + Returns a regex match containing library name after third_party + + Input one of: + //a/b/third_party/libname:c + //a/b/third_party/libname:c(//d/e/f:g) + //a/b/third_party/libname/c:d(//e/f/g:h) + + Outputs match with libname in group 1 or None if this is not a third_party + dependency. + """ + return re.match(THIRD_PARTY_LIB_REGEX, dep) + + @staticmethod + def _RunGN(buildfile_dir, target): + cmd = ['gn', 'desc', '--all', '--format=json', + os.path.abspath(buildfile_dir), target] + logging.debug("Running: %r", cmd) + output_json = subprocess.check_output(cmd, cwd=CHECKOUT_ROOT) + logging.debug("Output: %s", output_json) + return output_json + + @staticmethod + def _GetThirdPartyLibraries(buildfile_dir, target): + output = json.loads(LicenseBuilder._RunGN(buildfile_dir, target)) + libraries = set() + for target in output.values(): + third_party_matches = ( + LicenseBuilder._ParseLibrary(dep) for dep in target['deps']) + libraries |= set(match.group(1) for match in third_party_matches if match) + return libraries + + def GenerateLicenseText(self, output_dir): + # Get a list of third_party libs from gn. For fat libraries we must consider + # all architectures, hence the multiple buildfile directories. + third_party_libs = set() + for buildfile in self.buildfile_dirs: + for target in self.targets: + third_party_libs |= LicenseBuilder._GetThirdPartyLibraries( + buildfile, target) + assert len(third_party_libs) > 0 + + missing_licenses = third_party_libs - set(LIB_TO_LICENSES_DICT.keys()) + if missing_licenses: + error_msg = 'Missing licenses: %s' % ', '.join(missing_licenses) + logging.error(error_msg) + raise Exception(error_msg) + + # Put webrtc at the front of the list. + license_libs = sorted(third_party_libs) + license_libs.insert(0, 'webrtc') + + logging.info("List of licenses: %s", ', '.join(license_libs)) + + # Generate markdown. + output_license_file = open(os.path.join(output_dir, 'LICENSE.md'), 'w+') + for license_lib in license_libs: + if len(LIB_TO_LICENSES_DICT[license_lib]) == 0: + logging.info("Skipping compile time dependency: %s", license_lib) + continue # Compile time dependency + + output_license_file.write('# %s\n' % license_lib) + output_license_file.write('```\n') + for path in LIB_TO_LICENSES_DICT[license_lib]: + license_path = os.path.join(CHECKOUT_ROOT, path) + with open(license_path, 'r') as license_file: + license_text = cgi.escape(license_file.read(), quote=True) + output_license_file.write(license_text) + output_license_file.write('\n') + output_license_file.write('```\n\n') + + output_license_file.close() + + +def main(): + parser = argparse.ArgumentParser(description='Generate WebRTC LICENSE.md') + parser.add_argument('--verbose', action='store_true', default=False, + help='Debug logging.') + parser.add_argument('--target', required=True, action='append', default=[], + help='Name of the GN target to generate a license for') + parser.add_argument('output_dir', + help='Directory to output LICENSE.md to.') + parser.add_argument('buildfile_dirs', nargs="+", + help='Directories containing gn generated ninja files') + args = parser.parse_args() + + logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) + + builder = LicenseBuilder(args.buildfile_dirs, args.target) + builder.GenerateLicenseText(args.output_dir) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools_webrtc/libs/generate_licenses_test.py b/tools_webrtc/libs/generate_licenses_test.py new file mode 100755 index 0000000000..9ac541adf9 --- /dev/null +++ b/tools_webrtc/libs/generate_licenses_test.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# pylint: disable=relative-import,protected-access,unused-argument + +# Copyright 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import os +import sys + +SRC = os.path.abspath(os.path.join( + os.path.dirname((__file__)), os.pardir, os.pardir)) +sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) + +import mock +import unittest + +from generate_licenses import LicenseBuilder + + +class TestLicenseBuilder(unittest.TestCase): + @staticmethod + def _FakeRunGN(buildfile_dir, target): + return """ + { + "target1": { + "deps": [ + "//a/b/third_party/libname1:c", + "//a/b/third_party/libname2:c(//d/e/f:g)", + "//a/b/third_party/libname3/c:d(//e/f/g:h)", + "//a/b/not_third_party/c" + ] + } + } + """ + + def testParseLibrary(self): + self.assertEquals(LicenseBuilder._ParseLibrary( + '//a/b/third_party/libname1:c').group(1), + 'libname1') + self.assertEquals(LicenseBuilder._ParseLibrary( + '//a/b/third_party/libname2:c(d)').group(1), + 'libname2') + self.assertEquals(LicenseBuilder._ParseLibrary( + '//a/b/third_party/libname3/c:d(e)').group(1), + 'libname3') + self.assertEquals(LicenseBuilder._ParseLibrary('//a/b/not_third_party/c'), + None) + + @mock.patch('generate_licenses.LicenseBuilder._RunGN', _FakeRunGN) + def testGetThirdPartyLibraries(self): + self.assertEquals(LicenseBuilder._GetThirdPartyLibraries( + 'out/arm', 'target1'), + set(['libname1', 'libname2', 'libname3'])) + + +if __name__ == '__main__': + unittest.main() |