diff options
author | Jordan Bayles <jophba@chromium.org> | 2020-11-30 12:31:47 -0800 |
---|---|---|
committer | Jordan Bayles <jophba@chromium.org> | 2020-12-01 14:49:32 +0000 |
commit | e50a1d65fa41a9864e900e60bd7a9c59dfe4bb78 (patch) | |
tree | 6a15938c6e5c8e9891c3a8115ea6658f95c24a93 | |
parent | 4654f2b60f7a3fdb1a84d39d6504ba57be77f863 (diff) | |
download | openscreen-e50a1d65fa41a9864e900e60bd7a9c59dfe4bb78.tar.gz |
Add LICENSE file checking
This patch adds license file checking in Open Screen, by adding a new
licenses.py that iterates through third_party dependencies and returns
a list of licensing errors. This runs as part of PRESUBMIT.
This patch also includes README.chromium and LICENSE fixes for
third_party dependencies.
Finally, as part of refactoring I fixed the DEPS checker--before this
patch it fails to load modules so never runs on pre-submission.
Bug: b/173625891
Change-Id: I417e61f878dab809cf959d69480be749f9229128
Reviewed-on: https://chromium-review.googlesource.com/c/openscreen/+/2547807
Reviewed-by: Jordan Bayles <jophba@chromium.org>
Reviewed-by: mark a. foltz <mfoltz@chromium.org>
-rwxr-xr-x | PRESUBMIT.py | 218 | ||||
-rw-r--r-- | third_party/abseil/README.chromium | 13 | ||||
-rw-r--r-- | third_party/boringssl/README.chromium | 11 | ||||
-rw-r--r-- | third_party/chromium_quic/README.chromium | 13 | ||||
-rw-r--r-- | third_party/jsoncpp/README.chromium | 12 | ||||
-rw-r--r-- | third_party/libfuzzer/LICENSE.txt | 70 | ||||
-rw-r--r-- | third_party/libfuzzer/README.chromium | 9 | ||||
-rw-r--r-- | third_party/libprotobuf-mutator/README.chromium | 10 | ||||
-rw-r--r-- | third_party/mDNSResponder/README.chromium | 10 | ||||
-rw-r--r-- | third_party/mozilla/README.chromium | 7 | ||||
-rw-r--r-- | third_party/protobuf/README.chromium | 12 | ||||
-rw-r--r-- | third_party/tinycbor/README.chromium | 10 | ||||
-rw-r--r-- | third_party/zlib/LICENSE | 25 | ||||
-rw-r--r-- | third_party/zlib/README.chromium | 15 | ||||
-rwxr-xr-x | tools/licenses.py | 509 |
15 files changed, 843 insertions, 101 deletions
diff --git a/PRESUBMIT.py b/PRESUBMIT.py index dafbc5ea..07d567c5 100755 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -2,7 +2,21 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import os import re +import sys + +_REPO_PATH = os.path.dirname(os.path.realpath('__file__')) +_IMPORT_SUBFOLDERS = ['tools', os.path.join('buildtools', 'checkdeps')] + +# git-cl upload is not compatible with __init__.py based subfolder imports, so +# we extend the system path instead. +sys.path.extend(os.path.join(_REPO_PATH, p) for p in _IMPORT_SUBFOLDERS) + +import licenses +from checkdeps import DepsChecker +from cpp_checker import CppChecker +from rules import Rule # Rather than pass this to all of the checks, we override the global excluded # list with this one. @@ -22,25 +36,19 @@ _EXCLUDED_PATHS = ( ) +def _CheckLicenses(input_api, output_api): + """Checks third party licenses and returns a list of violations.""" + return [ + output_api.PresubmitError(v) for v in licenses.ScanThirdPartyDirs() + ] + + def _CheckDeps(input_api, output_api): - results = [] - import sys - original_sys_path = sys.path - try: - sys.path = sys.path + [input_api.os_path.join( - input_api.PresubmitLocalPath(), 'buildtools', 'checkdeps')] - import checkdeps - from cpp_checker import CppChecker - from rules import Rule - finally: - sys.path = original_sys_path - - deps_checker = checkdeps.DepsChecker(input_api.PresubmitLocalPath()) - deps_checker.CheckDirectory(input_api.PresubmitLocalPath()) - deps_results = deps_checker.results_formatter.GetResults() - for violation in deps_results: - results.append(output_api.PresubmitError(violation)) - return results + """Checks DEPS rules and returns a list of violations.""" + deps_checker = DepsChecker(input_api.PresubmitLocalPath()) + deps_checker.CheckDirectory(input_api.PresubmitLocalPath()) + deps_results = deps_checker.results_formatter.GetResults() + return [output_api.PresubmitError(v) for v in deps_results] # Matches Foo(Foo&&) when not followed by noexcept. @@ -49,7 +57,7 @@ _RE_PATTERN_MOVE_WITHOUT_NOEXCEPT = re.compile( def _CheckNoexceptOnMove(filename, clean_lines, linenum, error): - """Checks that move constructors are declared with 'noexcept'. + """Checks that move constructors are declared with 'noexcept'. Args: filename: The name of the current file. @@ -57,17 +65,19 @@ def _CheckNoexceptOnMove(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - # We only check headers as noexcept is meaningful on declarations, not - # definitions. This may skip some definitions in .cc files though. - if not filename.endswith('.h'): - return - - line = clean_lines.elided[linenum] - matched = _RE_PATTERN_MOVE_WITHOUT_NOEXCEPT.match(line) - if matched: - error(filename, linenum, 'runtime/noexcept', 4, - 'Move constructor of %s not declared \'noexcept\' in %s' % - (matched.group('classname'), matched.group(0).strip())) + # We only check headers as noexcept is meaningful on declarations, not + # definitions. This may skip some definitions in .cc files though. + if not filename.endswith('.h'): + return + + line = clean_lines.elided[linenum] + matched = _RE_PATTERN_MOVE_WITHOUT_NOEXCEPT.match(line) + if matched: + error( + filename, linenum, 'runtime/noexcept', 4, + 'Move constructor of {} not declared \'noexcept\' in {}'.format( + matched.group('classname'), + matched.group(0).strip())) # - We disable c++11 header checks since Open Screen allows them. # - We disable whitespace/braces because of various false positives. @@ -75,83 +85,89 @@ def _CheckNoexceptOnMove(filename, clean_lines, linenum, error): # enough to keep. # - We add a custom check for 'noexcept' usage. def _CheckChangeLintsClean(input_api, output_api): - """Checks that all '.cc' and '.h' files pass cpplint.py.""" - result = [] - - cpplint = input_api.cpplint - # Access to a protected member _XX of a client class - # pylint: disable=protected-access - cpplint._cpplint_state.ResetErrorCounts() - - cpplint._SetFilters('-build/c++11,-whitespace/braces') - files = [f.AbsoluteLocalPath() for f in input_api.AffectedSourceFiles(None)] - for file_name in files: - # 4 = verbose_level - cpplint.ProcessFile(file_name, 4, [_CheckNoexceptOnMove]) - - if cpplint._cpplint_state.error_count > 0: - if input_api.is_committing: - res_type = output_api.PresubmitError - else: - res_type = output_api.PresubmitPromptWarning - result = [res_type('Changelist failed cpplint.py check.')] - - return result + """Checks that all '.cc' and '.h' files pass cpplint.py.""" + cpplint = input_api.cpplint + # Access to a protected member _XX of a client class + # pylint: disable=protected-access + cpplint._cpplint_state.ResetErrorCounts() + + cpplint._SetFilters('-build/c++11,-whitespace/braces') + files = [ + f.AbsoluteLocalPath() for f in input_api.AffectedSourceFiles(None) + ] + CPPLINT_VERBOSE_LEVEL = 4 + for file_name in files: + cpplint.ProcessFile(file_name, CPPLINT_VERBOSE_LEVEL, + [_CheckNoexceptOnMove]) + + if cpplint._cpplint_state.error_count: + if input_api.is_committing: + res_type = output_api.PresubmitError + else: + res_type = output_api.PresubmitPromptWarning + return [res_type('Changelist failed cpplint.py check.')] + + return [] def _CommonChecks(input_api, output_api): - results = [] - # PanProjectChecks include: - # CheckLongLines (@ 80 cols) - # CheckChangeHasNoTabs - # CheckChangeHasNoStrayWhitespace - # CheckLicense - # CheckChangeWasUploaded (if committing) - # CheckChangeHasDescription - # CheckDoNotSubmitInDescription - # CheckDoNotSubmitInFiles - results.extend(input_api.canned_checks.PanProjectChecks( - input_api, output_api, owners_check=False)); - - # No carriage return characters, files end with one EOL (\n). - results.extend(input_api.canned_checks.CheckChangeHasNoCrAndHasOnlyOneEol( - input_api, output_api)); - - # Gender inclusivity - results.extend(input_api.canned_checks.CheckGenderNeutral( - input_api, output_api)) - - # TODO(bug) format required - results.extend(input_api.canned_checks.CheckChangeTodoHasOwner( - input_api, output_api)) - - # Linter. - results.extend(_CheckChangeLintsClean(input_api, output_api)) - - # clang-format - results.extend(input_api.canned_checks.CheckPatchFormatted( - input_api, output_api, bypass_warnings=False)) - - # GN formatting - results.extend(input_api.canned_checks.CheckGNFormatted( - input_api, output_api)) - - # buildtools/checkdeps - results.extend(_CheckDeps(input_api, output_api)) - return results + # PanProjectChecks include: + # CheckLongLines (@ 80 cols) + # CheckChangeHasNoTabs + # CheckChangeHasNoStrayWhitespace + # CheckLicense + # CheckChangeWasUploaded (if committing) + # CheckChangeHasDescription + # CheckDoNotSubmitInDescription + # CheckDoNotSubmitInFiles + results = input_api.canned_checks.PanProjectChecks(input_api, + output_api, + owners_check=False) + + # No carriage return characters, files end with one EOL (\n). + results.extend( + input_api.canned_checks.CheckChangeHasNoCrAndHasOnlyOneEol( + input_api, output_api)) + + # Gender inclusivity + results.extend( + input_api.canned_checks.CheckGenderNeutral(input_api, output_api)) + + # TODO(bug) format required + results.extend( + input_api.canned_checks.CheckChangeTodoHasOwner(input_api, output_api)) + + # Linter. + results.extend(_CheckChangeLintsClean(input_api, output_api)) + + # clang-format + results.extend( + input_api.canned_checks.CheckPatchFormatted(input_api, + output_api, + bypass_warnings=False)) + + # GN formatting + results.extend( + input_api.canned_checks.CheckGNFormatted(input_api, output_api)) + + # buildtools/checkdeps + results.extend(_CheckDeps(input_api, output_api)) + + # tools/licenses + results.extend(_CheckLicenses(input_api, output_api)) + + return results def CheckChangeOnUpload(input_api, output_api): - input_api.DEFAULT_FILES_TO_SKIP = _EXCLUDED_PATHS; - results = [] - results.extend(_CommonChecks(input_api, output_api)) - results.extend( - input_api.canned_checks.CheckChangedLUCIConfigs(input_api, output_api)) - return results + input_api.DEFAULT_FILES_TO_SKIP = _EXCLUDED_PATHS + # We always run the OnCommit checks, as well as some additional checks. + results = CheckChangeOnCommit(input_api, output_api) + results.extend( + input_api.canned_checks.CheckChangedLUCIConfigs(input_api, output_api)) + return results def CheckChangeOnCommit(input_api, output_api): - input_api.DEFAULT_FILES_TO_SKIP = _EXCLUDED_PATHS; - results = [] - results.extend(_CommonChecks(input_api, output_api)) - return results + input_api.DEFAULT_FILES_TO_SKIP = _EXCLUDED_PATHS + return _CommonChecks(input_api, output_api) diff --git a/third_party/abseil/README.chromium b/third_party/abseil/README.chromium new file mode 100644 index 00000000..fb55ac0f --- /dev/null +++ b/third_party/abseil/README.chromium @@ -0,0 +1,13 @@ +Name: Abseil +Short Name: absl +URL: https://github.com/abseil/abseil-cpp +License: Apache 2.0 +License File: src/LICENSE +Version: 0 +Security Critical: yes + +Description: +This directory contains the source code of Abseil for C++. Open Screen primarily +uses this library to essentially backport C++17 and C++20 features. The set of +files included from Abseil has been selected judiciously, and care should be +taken when adding dependencies on new Abseil files. diff --git a/third_party/boringssl/README.chromium b/third_party/boringssl/README.chromium new file mode 100644 index 00000000..adecfa89 --- /dev/null +++ b/third_party/boringssl/README.chromium @@ -0,0 +1,11 @@ +Name: BoringSSL +URL: https://boringssl.googlesource.com/boringssl +Version: git +License: BSDish +License File: src/LICENSE +License Android Compatible: yes +Security Critical: yes + +Description: +This is BoringSSL, a fork of OpenSSL. See +https://www.imperialviolet.org/2014/06/20/boringssl.html diff --git a/third_party/chromium_quic/README.chromium b/third_party/chromium_quic/README.chromium new file mode 100644 index 00000000..2570ceca --- /dev/null +++ b/third_party/chromium_quic/README.chromium @@ -0,0 +1,13 @@ +Name: Chromium QUIC +Short Name: chromium_quic +URL: https://chromium.googlesource.com/openscreen/quic.git +License: ISC +License File: src/LICENSE +Version: 0 +Security Critical: yes + +Description: +This directory contains the source code of Abseil for C++. Open Screen primarily +uses this library to essentially backport C++17 and C++20 features. The set of +files included from Abseil has been selected judiciously, and care should be +taken when adding dependencies on new Abseil files. diff --git a/third_party/jsoncpp/README.chromium b/third_party/jsoncpp/README.chromium new file mode 100644 index 00000000..3acbb5b6 --- /dev/null +++ b/third_party/jsoncpp/README.chromium @@ -0,0 +1,12 @@ +Name: jsoncpp +URL: https://github.com/open-source-parsers/jsoncpp +Version: 1.9.4 +License: MIT +License File: src/LICENSE +Security Critical: yes + +Description: +JsonCpp is used for parsing and generating JSON data. This +project is mirrored for Chrome from the public GitHub project, with a custom BUILD.gn +to allow for building with our Ninja + GN configuration. The main project uses +Meson or CMake for building.
\ No newline at end of file diff --git a/third_party/libfuzzer/LICENSE.txt b/third_party/libfuzzer/LICENSE.txt new file mode 100644 index 00000000..0fcf3f83 --- /dev/null +++ b/third_party/libfuzzer/LICENSE.txt @@ -0,0 +1,70 @@ +============================================================================== +LLVM Release License +============================================================================== +University of Illinois/NCSA +Open Source License + +Copyright (c) 2003-2015 University of Illinois at Urbana-Champaign. +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== +Copyrights and Licenses for Third Party Software Distributed with LLVM: +============================================================================== +The LLVM software contains code written by third parties. Such software will +have its own individual LICENSE.TXT file in the directory in which it appears. +This file will describe the copyrights, license, and restrictions which apply +to that code. + +The disclaimer of warranty in the University of Illinois Open Source License +applies to all code in the LLVM Distribution, and nothing in any of the +other licenses gives permission to use the names of the LLVM Team or the +University of Illinois to endorse or promote products derived from this +Software. + +The following pieces of software have additional or alternate copyrights, +licenses, and/or restrictions: + +Program Directory +------- --------- +Autoconf llvm/autoconf + llvm/projects/ModuleMaker/autoconf +Google Test llvm/utils/unittest/googletest +OpenBSD regex llvm/lib/Support/{reg*, COPYRIGHT.regex} +pyyaml tests llvm/test/YAMLParser/{*.data, LICENSE.TXT} +ARM contributions llvm/lib/Target/ARM/LICENSE.TXT +md5 contributions llvm/lib/Support/MD5.cpp llvm/include/llvm/Support/MD5.h
\ No newline at end of file diff --git a/third_party/libfuzzer/README.chromium b/third_party/libfuzzer/README.chromium new file mode 100644 index 00000000..3cb06feb --- /dev/null +++ b/third_party/libfuzzer/README.chromium @@ -0,0 +1,9 @@ +Name: libFuzzer +URL: http://llvm.org/docs/LibFuzzer.html +License: University of Illinois/NCSA Open Source +License File: LICENSE.txt +Security Critical: no + +Description: +Library for in-process coverage-guided fuzz testing (fuzzing) of other +libraries. diff --git a/third_party/libprotobuf-mutator/README.chromium b/third_party/libprotobuf-mutator/README.chromium new file mode 100644 index 00000000..7a3fccc4 --- /dev/null +++ b/third_party/libprotobuf-mutator/README.chromium @@ -0,0 +1,10 @@ +Name: libprotobuf-mutator +URL: https://github.com/google/libprotobuf-mutator +Version: 0 +License: Apache 2.0 +License File: src/LICENSE +Security Critical: no + +Description: +Library for protocol buffer mutation. Assistance library to in-process +coverage-guided fuzz testing (fuzzing). diff --git a/third_party/mDNSResponder/README.chromium b/third_party/mDNSResponder/README.chromium new file mode 100644 index 00000000..77469188 --- /dev/null +++ b/third_party/mDNSResponder/README.chromium @@ -0,0 +1,10 @@ +Name: mDNSResponder +URL: https://github.com/jevinskie/mDNSResponder +License: Apache License, Version 2.0 +License File: src/LICENSE +Security Critical: no + +Description: + +Pull from Apple Bonjour's MDNS/DNS-SD implementation. Will eventually be +replaced with our custom implementation, currently only used in osp. diff --git a/third_party/mozilla/README.chromium b/third_party/mozilla/README.chromium new file mode 100644 index 00000000..cb0eead2 --- /dev/null +++ b/third_party/mozilla/README.chromium @@ -0,0 +1,7 @@ +Name: url_parse +URL: http://mxr.mozilla.org/comm-central/source/mozilla/netwerk/base/src/nsURLParsers.cpp +Security Critical: yes +License: BSD and MPL 1.1/GPL 2.0/LGPL 2.1 +License File: LICENSE.txt +Description: +The file url_parse.cc is based on nsURLParsers.cc from Mozilla. diff --git a/third_party/protobuf/README.chromium b/third_party/protobuf/README.chromium new file mode 100644 index 00000000..3ee30647 --- /dev/null +++ b/third_party/protobuf/README.chromium @@ -0,0 +1,12 @@ +Name: Protocol Buffers +Short Name: protobuf +URL: https://github.com/google/protobuf +License: BSD +License File: src/LICENSE +Security Critical: yes + +Description: + +Protocol buffers are Google's language-neutral, platform-neutral, extensible +mechanism for serializing structured data. Open Screen uses protobufs primary +for message serialization.
\ No newline at end of file diff --git a/third_party/tinycbor/README.chromium b/third_party/tinycbor/README.chromium new file mode 100644 index 00000000..1532f3f3 --- /dev/null +++ b/third_party/tinycbor/README.chromium @@ -0,0 +1,10 @@ +Name: Tiny CBOR +Short Name: tinycbor +URL: https://chromium.googlesource.com/external/github.com/intel/tinycbor.git +License: MIT +License File: src/LICENSE +Security Critical: yes + +Description: +This directory contains the source code of TinyCGBOR, which is a binary object +representation library. diff --git a/third_party/zlib/LICENSE b/third_party/zlib/LICENSE new file mode 100644 index 00000000..91dc1c40 --- /dev/null +++ b/third_party/zlib/LICENSE @@ -0,0 +1,25 @@ +/* zlib.h -- interface of the 'zlib' general purpose compression library + version 1.2.11, January 15th, 2017 + + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + +*/ diff --git a/third_party/zlib/README.chromium b/third_party/zlib/README.chromium new file mode 100644 index 00000000..3082682b --- /dev/null +++ b/third_party/zlib/README.chromium @@ -0,0 +1,15 @@ +Name: zlib +Short Name: zlib +URL: http://zlib.net/ +Security Critical: yes +License: Custom license +License File: LICENSE +License Android Compatible: yes + +Description: +"A massively spiffy yet delicately unobtrusive compression library." + +zlib is a free, general-purpose, legally unencumbered lossless data-compression +library. zlib implements the "deflate" compression algorithm described by RFC +1951, which combines the LZ77 (Lempel-Ziv) algorithm with Huffman coding. zlib +also implements the zlib (RFC 1950) and gzip (RFC 1952) wrapper formats. diff --git a/tools/licenses.py b/tools/licenses.py new file mode 100755 index 00000000..98734550 --- /dev/null +++ b/tools/licenses.py @@ -0,0 +1,509 @@ +#!/usr/bin/env python3 +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Utility for checking and processing licensing information in third_party +directories. Copied from Chrome's tools/licenses.py. + +Usage: licenses.py <command> + +Commands: + scan scan third_party directories, verifying that we have licensing info + credits generate about:credits on stdout + +(You can also import this as a module.) +""" +from __future__ import print_function + +import argparse +import codecs +import json +import os +import shutil +import re +import subprocess +import sys +import tempfile + +# TODO(issuetracker.google.com/173766869): Remove Python2 checks/compatibility. +if sys.version_info.major == 2: + from cgi import escape +else: + from html import escape + +_REPOSITORY_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + +# Paths from the root of the tree to directories to skip. +PRUNE_PATHS = set([ + # Used for development and test, not in the shipping product. + os.path.join('third_party', 'llvm-build'), +]) + +# Directories we don't scan through. +PRUNE_DIRS = ('.git') + +# Directories where we check out directly from upstream, and therefore +# can't provide a README.chromium. Please prefer a README.chromium +# wherever possible. +SPECIAL_CASES = { + os.path.join('third_party', 'googletest'): { + "Name": "gtest", + "URL": "http://code.google.com/p/googletest", + "License": "BSD", + "License File": "NOT_SHIPPED", + } +} + +# Special value for 'License File' field used to indicate that the license file +# should not be used in about:credits. +NOT_SHIPPED = "NOT_SHIPPED" + + +def MakeDirectory(dir_path): + try: + os.makedirs(dir_path) + except OSError: + pass + + +def WriteDepfile(depfile_path, first_gn_output, inputs=None): + assert depfile_path != first_gn_output # http://crbug.com/646165 + assert not isinstance(inputs, string_types) # Easy mistake to make + inputs = inputs or [] + MakeDirectory(os.path.dirname(depfile_path)) + # Ninja does not support multiple outputs in depfiles. + with open(depfile_path, 'w') as depfile: + depfile.write(first_gn_output.replace(' ', '\\ ')) + depfile.write(': ') + depfile.write(' '.join(i.replace(' ', '\\ ') for i in inputs)) + depfile.write('\n') + + +class LicenseError(Exception): + """We raise this exception when a directory's licensing info isn't + fully filled out.""" + pass + + +def AbsolutePath(path, filename, root): + """Convert a path in README.chromium to be absolute based on the source + root.""" + if filename.startswith('/'): + # Absolute-looking paths are relative to the source root + # (which is the directory we're run from). + absolute_path = os.path.join(root, filename[1:]) + else: + absolute_path = os.path.join(root, path, filename) + if os.path.exists(absolute_path): + return absolute_path + return None + + +def ParseDir(path, root, require_license_file=True, optional_keys=None): + """Examine a third_party/foo component and extract its metadata.""" + # Parse metadata fields out of README.chromium. + # We examine "LICENSE" for the license file by default. + metadata = { + "License File": "LICENSE", # Relative path to license text. + "Name": None, # Short name (for header on about:credits). + "URL": None, # Project home page. + "License": None, # Software license. + } + + if optional_keys is None: + optional_keys = [] + + if path in SPECIAL_CASES: + metadata.update(SPECIAL_CASES[path]) + else: + # Try to find README.chromium. + readme_path = os.path.join(root, path, 'README.chromium') + if not os.path.exists(readme_path): + raise LicenseError("missing README.chromium or licenses.py " + "SPECIAL_CASES entry in %s\n" % path) + + for line in open(readme_path): + line = line.strip() + if not line: + break + for key in list(metadata.keys()) + optional_keys: + field = key + ": " + if line.startswith(field): + metadata[key] = line[len(field):] + + # Check that all expected metadata is present. + errors = [] + for key, value in metadata.items(): + if not value: + errors.append("couldn't find '" + key + "' line " + "in README.chromium or licences.py " + "SPECIAL_CASES") + + # Special-case modules that aren't in the shipping product, so don't need + # their license in about:credits. + if metadata["License File"] != NOT_SHIPPED: + # Check that the license file exists. + for filename in (metadata["License File"], "COPYING"): + license_path = AbsolutePath(path, filename, root) + if license_path is not None: + break + + if require_license_file and not license_path: + errors.append("License file not found. " + "Either add a file named LICENSE, " + "import upstream's COPYING if available, " + "or add a 'License File:' line to " + "README.chromium with the appropriate path.") + metadata["License File"] = license_path + + if errors: + raise LicenseError("Errors in %s:\n %s\n" % + (path, ";\n ".join(errors))) + return metadata + + +def ContainsFiles(path, root): + """Determines whether any files exist in a directory or in any of its + subdirectories.""" + for _, dirs, files in os.walk(os.path.join(root, path)): + if files: + return True + for prune_dir in PRUNE_DIRS: + if prune_dir in dirs: + dirs.remove(prune_dir) + return False + + +def FilterDirsWithFiles(dirs_list, root): + # If a directory contains no files, assume it's a DEPS directory for a + # project not used by our current configuration and skip it. + return [x for x in dirs_list if ContainsFiles(x, root)] + + +def FindThirdPartyDirs(prune_paths, root): + """Find all third_party directories underneath the source root.""" + third_party_dirs = set() + for path, dirs, files in os.walk(root): + path = path[len(root) + 1:] # Pretty up the path. + + # .gitignore ignores /out*/, so do the same here. + if path in prune_paths or path.startswith('out'): + dirs[:] = [] + continue + + # Prune out directories we want to skip. + # (Note that we loop over PRUNE_DIRS so we're not iterating over a + # list that we're simultaneously mutating.) + for skip in PRUNE_DIRS: + if skip in dirs: + dirs.remove(skip) + + if os.path.basename(path) == 'third_party': + # Add all subdirectories that are not marked for skipping. + for dir in dirs: + dirpath = os.path.join(path, dir) + if dirpath not in prune_paths: + third_party_dirs.add(dirpath) + + # Don't recurse into any subdirs from here. + dirs[:] = [] + continue + + return third_party_dirs + + +def FindThirdPartyDirsWithFiles(root): + third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS, root) + return FilterDirsWithFiles(third_party_dirs, root) + + +# Many builders do not contain 'gn' in their PATH, so use the GN binary from +# //buildtools. +def _GnBinary(): + exe = 'gn' + if sys.platform.startswith('linux'): + subdir = 'linux64' + elif sys.platform == 'darwin': + subdir = 'mac' + elif sys.platform == 'win32': + subdir, exe = 'win', 'gn.exe' + else: + raise RuntimeError("Unsupported platform '%s'." % sys.platform) + + return os.path.join(_REPOSITORY_ROOT, 'buildtools', subdir, exe) + + +def GetThirdPartyDepsFromGNDepsOutput(gn_deps, target_os): + """Returns third_party/foo directories given the output of "gn desc deps". + + Note that it always returns the direct sub-directory of third_party + where README.chromium and LICENSE files are, so that it can be passed to + ParseDir(). e.g.: + third_party/cld_3/src/src/BUILD.gn -> third_party/cld_3 + + It returns relative paths from _REPOSITORY_ROOT, not absolute paths. + """ + third_party_deps = set() + for absolute_build_dep in gn_deps.split(): + relative_build_dep = os.path.relpath(absolute_build_dep, + _REPOSITORY_ROOT) + m = re.search( + r'^((.+[/\\])?third_party[/\\][^/\\]+[/\\])(.+[/\\])?BUILD\.gn$', + relative_build_dep) + if not m: + continue + third_party_path = m.group(1) + if any(third_party_path.startswith(p + os.sep) for p in PRUNE_PATHS): + continue + third_party_deps.add(third_party_path[:-1]) + return third_party_deps + + +def FindThirdPartyDeps(gn_out_dir, gn_target, target_os): + if not gn_out_dir: + raise RuntimeError("--gn-out-dir is required if --gn-target is used.") + + # Generate gn project in temp directory and use it to find dependencies. + # Current gn directory cannot be used when we run this script in a gn action + # rule, because gn doesn't allow recursive invocations due to potential side + # effects. + tmp_dir = None + try: + tmp_dir = tempfile.mkdtemp(dir=gn_out_dir) + shutil.copy(os.path.join(gn_out_dir, "args.gn"), tmp_dir) + subprocess.check_output([_GnBinary(), "gen", tmp_dir]) + gn_deps = subprocess.check_output([ + _GnBinary(), "desc", tmp_dir, gn_target, "deps", "--as=buildfile", + "--all" + ]) + if isinstance(gn_deps, bytes): + gn_deps = gn_deps.decode("utf-8") + finally: + if tmp_dir and os.path.exists(tmp_dir): + shutil.rmtree(tmp_dir) + + return GetThirdPartyDepsFromGNDepsOutput(gn_deps, target_os) + + +def ScanThirdPartyDirs(root=None): + """Scan a list of directories and report on any problems we find.""" + if root is None: + root = os.getcwd() + third_party_dirs = FindThirdPartyDirsWithFiles(root) + + errors = [] + for path in sorted(third_party_dirs): + try: + metadata = ParseDir(path, root) + except LicenseError as e: + errors.append((path, e.args[0])) + continue + + return ['{}: {}'.format(path, error) for path, error in sorted(errors)] + + +def GenerateCredits(file_template_file, + entry_template_file, + output_file, + target_os, + gn_out_dir, + gn_target, + depfile=None): + """Generate about:credits.""" + + def EvaluateTemplate(template, env, escape=True): + """Expand a template with variables like {{foo}} using a + dictionary of expansions.""" + for key, val in env.items(): + if escape: + val = escape(val) + template = template.replace('{{%s}}' % key, val) + return template + + def MetadataToTemplateEntry(metadata, entry_template): + env = { + 'name': metadata['Name'], + 'url': metadata['URL'], + 'license': open(metadata['License File']).read(), + } + return { + 'name': metadata['Name'], + 'content': EvaluateTemplate(entry_template, env), + 'license_file': metadata['License File'], + } + + if gn_target: + third_party_dirs = FindThirdPartyDeps(gn_out_dir, gn_target, target_os) + + # Sanity-check to raise a build error if invalid gn_... settings are + # somehow passed to this script. + if not third_party_dirs: + raise RuntimeError("No deps found.") + else: + third_party_dirs = FindThirdPartyDirs(PRUNE_PATHS, _REPOSITORY_ROOT) + + if not file_template_file: + file_template_file = os.path.join(_REPOSITORY_ROOT, 'components', + 'about_ui', 'resources', + 'about_credits.tmpl') + if not entry_template_file: + entry_template_file = os.path.join(_REPOSITORY_ROOT, 'components', + 'about_ui', 'resources', + 'about_credits_entry.tmpl') + + entry_template = open(entry_template_file).read() + entries = [] + # Start from Chromium's LICENSE file + chromium_license_metadata = { + 'Name': 'The Chromium Project', + 'URL': 'http://www.chromium.org', + 'License File': os.path.join(_REPOSITORY_ROOT, 'LICENSE') + } + entries.append( + MetadataToTemplateEntry(chromium_license_metadata, entry_template)) + + entries_by_name = {} + for path in third_party_dirs: + try: + metadata = ParseDir(path, _REPOSITORY_ROOT) + except LicenseError: + # TODO(phajdan.jr): Convert to fatal error (http://crbug.com/39240). + continue + if metadata['License File'] == NOT_SHIPPED: + continue + + new_entry = MetadataToTemplateEntry(metadata, entry_template) + # Skip entries that we've already seen. + prev_entry = entries_by_name.setdefault(new_entry['name'], new_entry) + if prev_entry is not new_entry and ( + prev_entry['content'] == new_entry['content']): + continue + + entries.append(new_entry) + + entries.sort(key=lambda entry: (entry['name'].lower(), entry['content'])) + for entry_id, entry in enumerate(entries): + entry['content'] = entry['content'].replace('{{id}}', str(entry_id)) + + entries_contents = '\n'.join([entry['content'] for entry in entries]) + file_template = open(file_template_file).read() + template_contents = "<!-- Generated by licenses.py; do not edit. -->" + template_contents += EvaluateTemplate(file_template, + {'entries': entries_contents}, + escape=False) + + if output_file: + changed = True + try: + old_output = open(output_file, 'r').read() + if old_output == template_contents: + changed = False + except: + pass + if changed: + with open(output_file, 'w') as output: + output.write(template_contents) + else: + print(template_contents) + + if depfile: + assert output_file + # Add in build.ninja so that the target will be considered dirty when + # gn gen is run. Otherwise, it will fail to notice new files being + # added. This is still not perfect, as it will fail if no build files + # are changed, but a new README.chromium / LICENSE is added. This + # shouldn't happen in practice however. + license_file_list = (entry['license_file'] for entry in entries) + license_file_list = (os.path.relpath(p) for p in license_file_list) + license_file_list = sorted(set(license_file_list)) + WriteDepfile(depfile, output_file, license_file_list + ['build.ninja']) + + return True + + +def _ReadFile(path): + """Reads a file from disk. + Args: + path: The path of the file to read, relative to the root of the + repository. + Returns: + The contents of the file as a string. + """ + with codecs.open(os.path.join(_REPOSITORY_ROOT, path), 'r', 'utf-8') as f: + return f.read() + + +def GenerateLicenseFile(output_file, gn_out_dir, gn_target, target_os): + """Generate a plain-text LICENSE file which can be used when you ship a part + of Chromium code (specified by gn_target) as a stand-alone library + (e.g., //ios/web_view). + + The LICENSE file contains licenses of both Chromium and third-party + libraries which gn_target depends on. """ + + third_party_dirs = FindThirdPartyDeps(gn_out_dir, gn_target, target_os) + + # Start with Chromium's LICENSE file. + content = [_ReadFile('LICENSE')] + + # Add necessary third_party. + for directory in sorted(third_party_dirs): + metadata = ParseDir(directory, + _REPOSITORY_ROOT, + require_license_file=True) + license_file = metadata['License File'] + if license_file and license_file != NOT_SHIPPED: + content.append('-' * 20) + content.append(directory.split(os.sep)[-1]) + content.append('-' * 20) + content.append(_ReadFile(license_file)) + + content_text = '\n'.join(content) + + if output_file: + with codecs.open(output_file, 'w', 'utf-8') as output: + output.write(content_text) + else: + print(content_text) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--file-template', + help='Template HTML to use for the license page.') + parser.add_argument('--entry-template', + help='Template HTML to use for each license.') + parser.add_argument('--target-os', help='OS that this build is targeting.') + parser.add_argument('--gn-out-dir', + help='GN output directory for scanning dependencies.') + parser.add_argument('--gn-target', + help='GN target to scan for dependencies.') + parser.add_argument('command', + choices=['help', 'scan', 'credits', 'license_file']) + parser.add_argument('output_file', nargs='?') + parser.add_argument('--depfile', + help='Path to depfile (refer to `gn help depfile`)') + args = parser.parse_args() + + if args.command == 'scan': + if not ScanThirdPartyDirs(): + return 1 + elif args.command == 'credits': + if not GenerateCredits(args.file_template, args.entry_template, + args.output_file, args.target_os, + args.gn_out_dir, args.gn_target, args.depfile): + return 1 + elif args.command == 'license_file': + try: + GenerateLicenseFile(args.output_file, args.gn_out_dir, + args.gn_target, args.target_os) + except LicenseError as e: + print("Failed to parse README.chromium: {}".format(e)) + return 1 + else: + print(__doc__) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) |