diff options
-rw-r--r-- | .gitignore | 2 | ||||
-rw-r--r-- | DEPS | 14 | ||||
-rwxr-xr-x | PRESUBMIT.py | 113 | ||||
-rw-r--r-- | THIRD_PARTY_DEPS | 97 | ||||
-rw-r--r-- | third_party/.gitignore | 64 | ||||
-rw-r--r-- | third_party/OWNERS | 7 | ||||
-rw-r--r-- | third_party/binutils/.gitignore | 8 | ||||
-rw-r--r-- | third_party/instrumented_libraries/binaries/.gitignore | 2 | ||||
-rw-r--r-- | third_party/opus/src/.gitignore | 83 | ||||
-rw-r--r-- | third_party/opus/src/win32/.gitignore | 26 | ||||
-rw-r--r-- | third_party/protobuf/.gitignore | 171 | ||||
-rw-r--r-- | third_party/protobuf/csharp/.gitignore | 31 | ||||
-rw-r--r-- | third_party/protobuf/objectivec/.gitignore | 23 | ||||
-rw-r--r-- | third_party/protobuf/ruby/.gitignore | 8 | ||||
-rwxr-xr-x | tools_webrtc/autoroller/roll_deps.py | 160 | ||||
-rwxr-xr-x | tools_webrtc/autoroller/unittests/roll_deps_test.py | 8 |
16 files changed, 734 insertions, 83 deletions
diff --git a/.gitignore b/.gitignore index bf937369ae..ecd89ce27e 100644 --- a/.gitignore +++ b/.gitignore @@ -45,7 +45,7 @@ /mojo /out /testing -/third_party +/third_party_chromium /tools /tools_webrtc/android/profiling/flamegraph /tools_webrtc/android/profiling/simpleperf @@ -2,6 +2,9 @@ vars = { 'chromium_git': 'https://chromium.googlesource.com', + # Used by the WebRTC DEPS autoroller to update third_party/. If you need to run autoroller localy, + # you can set it via custom_vars section in the .gclient file. + 'roll_chromium_into_webrtc': False, # By default, we should check out everything needed to run on the main # chromium waterfalls. More info at: crbug.com/570091. 'checkout_configuration': 'default', @@ -37,6 +40,10 @@ vars = { # the commit queue can handle CLs rolling HarfBuzz # and whatever else without interference from each other. 'harfbuzz_revision': '957e7756634a4fdf1654041e20e883cf964ecac9', + # Three lines of non-changing comments so that + # the commit queue can handle CLs rolling Chromium third_party + # and whatever else without interference from each other. + 'chromium_third_party_revision': '4e16929f465a47942875a80da0140bfaa59e99fb', } deps = { # TODO(kjellander): Move this to be Android-only once the libevent dependency @@ -59,8 +66,11 @@ deps = { }, 'src/testing': Var('chromium_git') + '/chromium/src/testing' + '@' + '519bd6bd8883f17137857e86cc73491d39415057', - 'src/third_party': - Var('chromium_git') + '/chromium/src/third_party' + '@' + '332e6754f377ca77a48eeeb139ee8d1c2f3ca739', + # This entry is used for chromium third_party rolling into webrtc third_party only. + 'src/third_party_chromium': { + 'url': Var('chromium_git') + '/chromium/src/third_party' + '@' + Var('chromium_third_party_revision'), + 'condition': 'roll_chromium_into_webrtc', + }, 'src/third_party/android_ndk': { 'url': Var('chromium_git') + '/android_ndk.git' + '@' + '5cd86312e794bdf542a3685c6f10cbb96072990b', 'condition': 'checkout_android', diff --git a/PRESUBMIT.py b/PRESUBMIT.py index 5a40612bfb..abb85825a3 100755 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -40,6 +40,7 @@ CPPLINT_BLACKLIST = [ 'test', 'tools_webrtc', 'voice_engine', + 'third_party', ] # These filters will always be removed, even if the caller specifies a filter @@ -180,12 +181,15 @@ def CheckNativeApiHeaderChanges(input_api, output_api): return [] -def CheckNoIOStreamInHeaders(input_api, output_api): +def CheckNoIOStreamInHeaders(input_api, output_api, + source_file_filter): """Checks to make sure no .h files include <iostream>.""" files = [] pattern = input_api.re.compile(r'^#include\s*<iostream>', input_api.re.MULTILINE) - for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile): + file_filter = lambda x: (input_api.FilterSourceFile(x) + and source_file_filter(x)) + for f in input_api.AffectedSourceFiles(file_filter): if not f.LocalPath().endswith('.h'): continue contents = input_api.ReadFile(f) @@ -201,12 +205,15 @@ def CheckNoIOStreamInHeaders(input_api, output_api): return [] -def CheckNoPragmaOnce(input_api, output_api): +def CheckNoPragmaOnce(input_api, output_api, + source_file_filter): """Make sure that banned functions are not used.""" files = [] pattern = input_api.re.compile(r'^#pragma\s+once', input_api.re.MULTILINE) - for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile): + file_filter = lambda x: (input_api.FilterSourceFile(x) + and source_file_filter(x)) + for f in input_api.AffectedSourceFiles(file_filter): if not f.LocalPath().endswith('.h'): continue contents = input_api.ReadFile(f) @@ -221,13 +228,15 @@ def CheckNoPragmaOnce(input_api, output_api): return [] -def CheckNoFRIEND_TEST(input_api, output_api): # pylint: disable=invalid-name +def CheckNoFRIEND_TEST(input_api, output_api, # pylint: disable=invalid-name + source_file_filter): """Make sure that gtest's FRIEND_TEST() macro is not used, the FRIEND_TEST_ALL_PREFIXES() macro from testsupport/gtest_prod_util.h should be used instead since that allows for FLAKY_, FAILS_ and DISABLED_ prefixes.""" problems = [] - file_filter = lambda f: f.LocalPath().endswith(('.cc', '.h')) + file_filter = lambda f: (f.LocalPath().endswith(('.cc', '.h')) + and source_file_filter(f)) for f in input_api.AffectedFiles(file_filter=file_filter): for line_num, line in f.ChangedContents(): if 'FRIEND_TEST(' in line: @@ -249,7 +258,7 @@ def IsLintBlacklisted(blacklist_paths, file_path): def CheckApprovedFilesLintClean(input_api, output_api, - source_file_filter=None): + source_file_filter=None): """Checks that all new or non-blacklisted .cc and .h files pass cpplint.py. This check is based on CheckChangeLintsClean in depot_tools/presubmit_canned_checks.py but has less filters and only checks @@ -408,7 +417,8 @@ def _ReportErrorFileAndLineNumber(filename, line_num): def CheckNoStreamUsageIsAdded(input_api, output_api, - error_formatter=_ReportErrorFileAndLineNumber): + error_formatter=_ReportErrorFileAndLineNumber, + source_file_filter): """Make sure that no more dependencies on stringstream are added.""" error_msg = ('Usage of <sstream>, <istream> and <ostream> in WebRTC is ' 'deprecated.\n' @@ -433,7 +443,9 @@ def CheckNoStreamUsageIsAdded(input_api, output_api, usage_re = input_api.re.compile(r'std::(w|i|o|io|wi|wo|wio)(string)*stream') no_presubmit_re = input_api.re.compile( r'// no-presubmit-check TODO\(webrtc:8982\)') - for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile): + file_filter = lambda x: (input_api.FilterSourceFile(x) + and source_file_filter(x)) + for f in input_api.AffectedSourceFiles(file_filter): if f.LocalPath() == 'PRESUBMIT.py': continue for line_num, line in f.ChangedContents(): @@ -482,13 +494,14 @@ def CheckCheckIncludesIsNotUsed(gn_files, output_api): line_number))) return result -def CheckGnChanges(input_api, output_api): - source_file_filter = lambda x: input_api.FilterSourceFile( +def CheckGnChanges(input_api, output_api, source_file_filter): + file_filter = lambda x: (input_api.FilterSourceFile( x, white_list=(r'.+\.(gn|gni)$',), black_list=(r'.*/presubmit_checks_lib/testdata/.*',)) + and source_file_filter(x)) gn_files = [] - for f in input_api.AffectedSourceFiles(source_file_filter): + for f in input_api.AffectedSourceFiles(file_filter): gn_files.append(f) result = [] @@ -517,7 +530,7 @@ def CheckGnGen(input_api, output_api): long_text='\n\n'.join(errors))] return [] -def CheckUnwantedDependencies(input_api, output_api): +def CheckUnwantedDependencies(input_api, output_api, source_file_filter): """Runs checkdeps on #include statements added in this change. Breaking - rules is an error, breaking ! rules is a warning. @@ -539,7 +552,7 @@ def CheckUnwantedDependencies(input_api, output_api): from rules import Rule added_includes = [] - for f in input_api.AffectedFiles(): + for f in input_api.AffectedFiles(file_filter=source_file_filter): if not CppChecker.IsCppFile(f.LocalPath()): continue @@ -620,11 +633,12 @@ def CheckChangeHasBugField(input_api, output_api): ' * https://bugs.webrtc.org - reference it using Bug: webrtc:XXXX\n' ' * https://crbug.com - reference it using Bug: chromium:XXXXXX')] -def CheckJSONParseErrors(input_api, output_api): +def CheckJSONParseErrors(input_api, output_api, source_file_filter): """Check that JSON files do not contain syntax errors.""" def FilterFile(affected_file): - return input_api.os_path.splitext(affected_file.LocalPath())[1] == '.json' + return (input_api.os_path.splitext(affected_file.LocalPath())[1] == '.json' + and source_file_filter(affected_file)) def GetJSONParseError(input_api, filename): try: @@ -670,12 +684,15 @@ def RunPythonTests(input_api, output_api): return input_api.RunTests(tests, parallel=True) -def CheckUsageOfGoogleProtobufNamespace(input_api, output_api): +def CheckUsageOfGoogleProtobufNamespace(input_api, output_api, + source_file_filter): """Checks that the namespace google::protobuf has not been used.""" files = [] pattern = input_api.re.compile(r'google::protobuf') proto_utils_path = os.path.join('rtc_base', 'protobuf_utils.h') - for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile): + file_filter = lambda x: (input_api.FilterSourceFile(x) + and source_file_filter(x)) + for f in input_api.AffectedSourceFiles(file_filter): if f.LocalPath() in [proto_utils_path, 'PRESUBMIT.py']: continue contents = input_api.ReadFile(f) @@ -752,8 +769,11 @@ def CommonChecks(input_api, output_api): objc_filter_list = (r'.+\.m$', r'.+\.mm$', r'.+objc\/.+\.h$') # Skip long-lines check for DEPS and GN files. build_file_filter_list = (r'.+\.gn$', r'.+\.gni$', 'DEPS') + # Also we will skip most checks for third_party directory. + third_party_filter_list = (r'^third_party[\\\/].+',) eighty_char_sources = lambda x: input_api.FilterSourceFile(x, - black_list=build_file_filter_list + objc_filter_list) + black_list=build_file_filter_list + objc_filter_list + + third_party_filter_list) hundred_char_sources = lambda x: input_api.FilterSourceFile(x, white_list=objc_filter_list) results.extend(input_api.canned_checks.CheckLongLines( @@ -762,26 +782,38 @@ def CommonChecks(input_api, output_api): input_api, output_api, maxlen=100, source_file_filter=hundred_char_sources)) + non_third_party_sources = lambda x: input_api.FilterSourceFile(x, + black_list=third_party_filter_list) results.extend(input_api.canned_checks.CheckChangeHasNoTabs( - input_api, output_api)) + input_api, output_api, source_file_filter=non_third_party_sources)) results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace( - input_api, output_api)) + input_api, output_api, source_file_filter=non_third_party_sources)) results.extend(input_api.canned_checks.CheckAuthorizedAuthor( input_api, output_api)) results.extend(input_api.canned_checks.CheckChangeTodoHasOwner( - input_api, output_api)) + input_api, output_api, source_file_filter=non_third_party_sources)) results.extend(CheckNativeApiHeaderChanges(input_api, output_api)) - results.extend(CheckNoIOStreamInHeaders(input_api, output_api)) - results.extend(CheckNoPragmaOnce(input_api, output_api)) - results.extend(CheckNoFRIEND_TEST(input_api, output_api)) - results.extend(CheckGnChanges(input_api, output_api)) - results.extend(CheckUnwantedDependencies(input_api, output_api)) - results.extend(CheckJSONParseErrors(input_api, output_api)) + results.extend(CheckNoIOStreamInHeaders( + input_api, output_api, source_file_filter=non_third_party_sources)) + results.extend(CheckNoPragmaOnce( + input_api, output_api, source_file_filter=non_third_party_sources)) + results.extend(CheckNoFRIEND_TEST( + input_api, output_api, source_file_filter=non_third_party_sources)) + results.extend(CheckGnChanges( + input_api, output_api, source_file_filter=non_third_party_sources)) + results.extend(CheckUnwantedDependencies( + input_api, output_api, source_file_filter=non_third_party_sources)) + results.extend(CheckJSONParseErrors( + input_api, output_api, source_file_filter=non_third_party_sources)) results.extend(RunPythonTests(input_api, output_api)) - results.extend(CheckUsageOfGoogleProtobufNamespace(input_api, output_api)) - results.extend(CheckOrphanHeaders(input_api, output_api)) - results.extend(CheckNewlineAtTheEndOfProtoFiles(input_api, output_api)) - results.extend(CheckNoStreamUsageIsAdded(input_api, output_api)) + results.extend(CheckUsageOfGoogleProtobufNamespace( + input_api, output_api, source_file_filter=non_third_party_sources)) + results.extend(CheckOrphanHeaders( + input_api, output_api, source_file_filter=non_third_party_sources)) + results.extend(CheckNewlineAtTheEndOfProtoFiles( + input_api, output_api, source_file_filter=non_third_party_sources)) + results.extend(CheckNoStreamUsageIsAdded( + input_api, output_api, source_file_filter=non_third_party_sources)) return results @@ -811,7 +843,7 @@ def CheckChangeOnCommit(input_api, output_api): return results -def CheckOrphanHeaders(input_api, output_api): +def CheckOrphanHeaders(input_api, output_api, source_file_filter): # We need to wait until we have an input_api object and use this # roundabout construct to import prebubmit_checks_lib because this file is # eval-ed and thus doesn't have __file__. @@ -825,9 +857,9 @@ def CheckOrphanHeaders(input_api, output_api): from check_orphan_headers import GetBuildGnPathFromFilePath from check_orphan_headers import IsHeaderInBuildGn - source_file_filter = lambda x: input_api.FilterSourceFile( - x, black_list=orphan_blacklist) - for f in input_api.AffectedSourceFiles(source_file_filter): + file_filter = lambda x: input_api.FilterSourceFile( + x, black_list=orphan_blacklist) and source_file_filter(x) + for f in input_api.AffectedSourceFiles(file_filter): if f.LocalPath().endswith('.h'): file_path = os.path.abspath(f.LocalPath()) root_dir = os.getcwd() @@ -840,13 +872,14 @@ def CheckOrphanHeaders(input_api, output_api): return results -def CheckNewlineAtTheEndOfProtoFiles(input_api, output_api): +def CheckNewlineAtTheEndOfProtoFiles(input_api, output_api, + source_file_filter): """Checks that all .proto files are terminated with a newline.""" error_msg = 'File {} must end with exactly one newline.' results = [] - source_file_filter = lambda x: input_api.FilterSourceFile( - x, white_list=(r'.+\.proto$',)) - for f in input_api.AffectedSourceFiles(source_file_filter): + file_filter = lambda x: input_api.FilterSourceFile( + x, white_list=(r'.+\.proto$',)) and source_file_filter(x) + for f in input_api.AffectedSourceFiles(file_filter): file_path = f.LocalPath() with open(file_path) as f: lines = f.readlines() diff --git a/THIRD_PARTY_DEPS b/THIRD_PARTY_DEPS new file mode 100644 index 0000000000..d09c978b9b --- /dev/null +++ b/THIRD_PARTY_DEPS @@ -0,0 +1,97 @@ +DEPS = [ + # Common + 'binutils', + 'boringssl', + 'ced', + 'freetype', + 'googletest', + 'harfbuzz-ng', + 'instrumented_libraries', + 'jsoncpp', + 'libFuzzer', + 'libpng', + 'libvpx', + 'mockito', + 'openh264', + 'opus', + 'protobuf', + 'requests', + 'rnnoise', + 'usrsctp', + 'yasm', + 'zlib', + 'colorama', + + # These common deps will be synced by gclient: + #'depot_tools', + #'ffmpeg', + #'icu', + #'libjpeg_turbo', + #'libsrtp', + #'libyuv', + #'llvm-build', + #'lss', + #'openmax_dl', + #'catapult', + #'gtest-parallel', + + # Windows specific deps will be synced by gclient: + #'syzygy', + #'winsdk_samples', + + # Android specific deps + # compile time deps + 'accessibility_test_framework', + 'android_platform', + 'android_support_test_runner', + 'apk-patch-size-estimator', + 'ashmem', + 'auto', + 'bazel', + 'bouncycastle', + 'breakpad', + 'byte_buddy', + 'closure_compiler', + 'errorprone', + 'espresso', + 'eu-strip', + 'gson', + 'guava', + 'hamcrest', + 'icu4j', + 'ijar', + 'intellij', + 'javax_inject', + 'jinja2', + 'jsr-305', + 'junit', + 'libxml', + 'markupsafe', + 'modp_b64', + 'objenesis', + 'ow2_asm', + 'robolectric', + 'sqlite4java', + 'tcmalloc', + 'ub-uiautomator', + 'xstream', + # test time deps + 'proguard', + + 'android_system_sdk', + # These Android specific deps will be synced by gclient: + #'android_ndk', + #'android_tools', + #'findbugs', + + # Mac and iOS specific deps + 'ocmock', + + # List of files to sync + 'BUILD.gn', + 'DEPS', + 'libjpeg.gni', + 'PRESUBMIT.py', + 'README.chromium', + 'README.chromium.template', +] diff --git a/third_party/.gitignore b/third_party/.gitignore new file mode 100644 index 0000000000..566e8a3918 --- /dev/null +++ b/third_party/.gitignore @@ -0,0 +1,64 @@ +# mirror in DEPS. Without it, a lot is wiped and re-downloaded for each sync. +/findbugs/ +/gtest-parallel/ +/winsdk_samples/ +/accessibility_test_framework/lib/ +/android_ndk/ +/android_protobuf/src +/android_support_test_runner/lib/ +/android_tools/ +/android_tools_internal/ +/apk-patch-size-estimator/lib/ +/auto/src +/bazel/desugar/Desugar.jar +/boringssl/src +/bouncycastle/lib/ +/byte_buddy/lib/ +/catapult +/ced/src +/colorama/src +/depot_tools +/errorprone/lib +/espresso/lib/ +/ffmpeg +/freetype/src +/gnu_binutils/ +/googletest/src +/gson/lib/ +/guava/lib/ +/hamcrest/lib/ +/harfbuzz-ng/src +/icu +/icu4j/lib/ +/intellij/lib/ +/instrumented_libraries/scripts/*.tgz +/instrumented_libraries/scripts/out/* +/javax_inject/lib/ +/jsoncpp/source +/jsr-305/src +/junit/src +/libFuzzer/src +/libprotobuf-mutator/src +/libjpeg_turbo +/libsrtp +/libvpx/source/libvpx +/libyuv +/llvm-build +/llvm-build-tools +/lss +/mockito/src +/objenesis/lib/ +/openmax_dl/ +/openh264/src +/ow2_asm/lib/ +/requests/src +/robolectric/lib/ +/robolectric/robolectric +/sqlite4java/lib/ +/ub-uiautomator/lib +/usrsctp/usrsctplib +/xstream/lib/ +/yasm/binaries +/yasm/generate_files.xml +/yasm/source/patched-yasm +/yasm/yasm.xml diff --git a/third_party/OWNERS b/third_party/OWNERS new file mode 100644 index 0000000000..908fabb1d7 --- /dev/null +++ b/third_party/OWNERS @@ -0,0 +1,7 @@ +phoglund@webrtc.org +titovartem@webrtc.org +buildbot@webrtc.org + + + +per-file .gitignore=* diff --git a/third_party/binutils/.gitignore b/third_party/binutils/.gitignore new file mode 100644 index 0000000000..5605b2f711 --- /dev/null +++ b/third_party/binutils/.gitignore @@ -0,0 +1,8 @@ +binutils-* +*-chroot-* +output-* +Linux_ia32/*stamp* +Linux_ia32/*tar.bz2 +Linux_x64/*stamp* +Linux_x64/*tar.bz2 +*/Release diff --git a/third_party/instrumented_libraries/binaries/.gitignore b/third_party/instrumented_libraries/binaries/.gitignore new file mode 100644 index 0000000000..d177d9a553 --- /dev/null +++ b/third_party/instrumented_libraries/binaries/.gitignore @@ -0,0 +1,2 @@ +# Ignore downloaded binaries. +*.tgz diff --git a/third_party/opus/src/.gitignore b/third_party/opus/src/.gitignore new file mode 100644 index 0000000000..33127c9fe0 --- /dev/null +++ b/third_party/opus/src/.gitignore @@ -0,0 +1,83 @@ +Doxyfile +Makefile +Makefile.in +TAGS +aclocal.m4 +autom4te.cache +*.kdevelop.pcs +*.kdevses +compile +config.guess +config.h +config.h.in +config.log +config.status +config.sub +configure +depcomp +INSTALL +install-sh +.deps +.libs +.dirstamp +*.a +*.exe +*.la +*-gnu.S +testcelt +libtool +ltmain.sh +missing +m4/libtool.m4 +m4/ltoptions.m4 +m4/ltsugar.m4 +m4/ltversion.m4 +m4/lt~obsolete.m4 +opus_compare +opus_demo +repacketizer_demo +stamp-h1 +test-driver +*.sw* +*.o +*.lo +*.pc +*.tar.gz +*~ +tests/*test +tests/test_opus_api +tests/test_opus_decode +tests/test_opus_encode +tests/test_opus_padding +celt/arm/armopts.s +celt/dump_modes/dump_modes +celt/tests/test_unit_cwrs32 +celt/tests/test_unit_dft +celt/tests/test_unit_entropy +celt/tests/test_unit_laplace +celt/tests/test_unit_mathops +celt/tests/test_unit_mdct +celt/tests/test_unit_rotation +celt/tests/test_unit_types +doc/doxygen_sqlite3.db +doc/doxygen-build.stamp +doc/html +doc/latex +doc/man +package_version +version.h +celt/Debug +celt/Release +celt/x64 +silk/Debug +silk/Release +silk/x64 +silk/fixed/Debug +silk/fixed/Release +silk/fixed/x64 +silk/float/Debug +silk/float/Release +silk/float/x64 +src/Debug +src/Release +src/x64 diff --git a/third_party/opus/src/win32/.gitignore b/third_party/opus/src/win32/.gitignore new file mode 100644 index 0000000000..c17feab73d --- /dev/null +++ b/third_party/opus/src/win32/.gitignore @@ -0,0 +1,26 @@ +# Visual Studio ignores +[Dd]ebug/ +[Dd]ebugDLL/ +[Dd]ebugDLL_fixed/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleaseDLL/ +[Rr]eleaseDLL_fixed/ +[Rr]eleases/ +*.manifest +*.lastbuildstate +*.lib +*.log +*.idb +*.ipdb +*.ilk +*.iobj +*.obj +*.opensdf +*.pdb +*.sdf +*.suo +*.tlog +*.vcxproj.user +*.vc.db +*.vc.opendb diff --git a/third_party/protobuf/.gitignore b/third_party/protobuf/.gitignore new file mode 100644 index 0000000000..51d092c724 --- /dev/null +++ b/third_party/protobuf/.gitignore @@ -0,0 +1,171 @@ +# autogen.sh-generated files +Makefile.in +src/Makefile.in +config.guess +config.h.in +config.sub +configure +depcomp +install-sh +ltmain.sh +missing + +aclocal.m4 +m4/libtool.m4 +m4/ltoptions.m4 +m4/ltsugar.m4 +m4/ltversion.m4 +m4/lt~obsolete.m4 +autom4te.cache + +# downloaded files +gmock + +# in-tree configure-generated files +Makefile +src/Makefile +/config.h +config.log +config.status + +libtool +protobuf-lite.pc +protobuf.pc +.deps +stamp-h1 + +# in-tree build products +*.o +*.lo +*.la +src/.libs +*.so + +.dirstamp + +any_test.pb.* +map*unittest.pb.* +unittest*.pb.* +cpp_test*.pb.* +src/google/protobuf/util/**/*.pb.cc +src/google/protobuf/util/**/*.pb.h + +*.pyc +*.egg-info +*_pb2.py +python/*.egg +python/.eggs/ +python/.tox +python/build/ +python/google/protobuf/compiler/ +python/google/protobuf/util/ + +src/js_embed +src/protoc +src/unittest_proto_middleman + +# vim generated +*.swp + +# Generated test scaffolding +src/no_warning_test.cc +src/no-warning-test +src/protobuf*-test +src/test_plugin +src/testzip.* +src/zcg*zip +ar-lib + +test-driver +compile + +src/**/*.log +src/**/*.trs + +# JavaBuild output. +java/core/target +java/util/target +javanano/target +java/.idea +java/**/*.iml + +# Windows native output. +cmake/build +build_msvc + +# NuGet packages: we want the repository configuration, but not the +# packages themselves. +/csharp/src/packages/*/ + +# OS X's Finder creates these for state about opened windows/etc. +**/.DS_Store + +# Cocoapods artifacts +# Podfile.lock and the workspace file are tracked, to ease deleting them. That's +# needed to trigger "pod install" to rerun the preinstall commands. +Pods/ + +# Comformance test output +conformance/.libs/ +conformance/com/ +conformance/conformance-cpp +conformance/conformance-csharp +conformance/conformance-java +conformance/conformance-objc +conformance/conformance-test-runner +conformance/conformance.pb.cc +conformance/conformance.pb.h +conformance/Conformance.pbobjc.h +conformance/Conformance.pbobjc.m +conformance/conformance_pb.js +conformance/conformance_pb.rb +conformance/failing_tests.txt +conformance/google/ +conformance/google-protobuf/ +conformance/javac_middleman +conformance/lite/ +conformance/nonexistent_tests.txt +conformance/protoc_middleman +conformance/succeeding_tests.txt +conformance/Conformance/ +conformance/GPBMetadata/ +conformance/Google/ +conformance/Protobuf_test_messages/ +conformance/conformance-php +conformance/conformance-php-c + +# php test output +composer.lock +php/tests/generated/ +php/tests/old_protoc +php/tests/protobuf/ +php/ext/google/protobuf/.libs/ +php/ext/google/protobuf/Makefile.fragments +php/ext/google/protobuf/Makefile.global +php/ext/google/protobuf/Makefile.objects +php/ext/google/protobuf/acinclude.m4 +php/ext/google/protobuf/build/ +php/ext/google/protobuf/config.h +php/ext/google/protobuf/config.nice +php/ext/google/protobuf/configure.in +php/ext/google/protobuf/mkinstalldirs +php/ext/google/protobuf/run-tests.php +vendor/ + +# JavaScript artifacts +js/commonjs_out/ +js/compatibility_tests/v3.0.0/commonjs_out* +js/compatibility_tests/v3.0.0/protoc +js/compatibility_tests/v3.0.0/testproto_libs1.js +js/compatibility_tests/v3.0.0/testproto_libs1_new.js +js/compatibility_tests/v3.0.0/testproto_libs2.js +js/compatibility_tests/v3.0.0/testproto_libs2_new.js +js/deps.js +js/google-protobuf.js +js/google/ +js/node_modules/ +js/testproto_libs1.js +js/testproto_libs2.js + +# Ignore the bazel symlinks +/bazel-* diff --git a/third_party/protobuf/csharp/.gitignore b/third_party/protobuf/csharp/.gitignore new file mode 100644 index 0000000000..8ba8849985 --- /dev/null +++ b/third_party/protobuf/csharp/.gitignore @@ -0,0 +1,31 @@ +# Output +bin +obj +project.lock.json +TestResult.xml + +# Possibly legacy now? +mono/bin +mono/tmp +mono/protoc +build_output +build_temp +build/msbuild*.log +lib/Microsoft.Silverlight.Testing +lib/NUnit + +# +# Untracked files +# +.vs +*.user +*.suo +*.nupkg +_ReSharper.* +*.sln.cache +mono/TestResult.xml +mono/.libs +mono/*.exe +mono/*.dll +lib/protoc.exe +*.ncrunch* diff --git a/third_party/protobuf/objectivec/.gitignore b/third_party/protobuf/objectivec/.gitignore new file mode 100644 index 0000000000..f786ffce82 --- /dev/null +++ b/third_party/protobuf/objectivec/.gitignore @@ -0,0 +1,23 @@ +## Build generated +build/ +DerivedData/ + +## Various settings +*.pbxuser +!default.pbxuser +*.mode1v3 +!default.mode1v3 +*.mode2v3 +!default.mode2v3 +*.perspectivev3 +!default.perspectivev3 +xcuserdata/ + +## Other +*.moved-aside +*.xccheckout +*.xcscmblueprint + +## Obj-C/Swift specific +*.hmap +*.ipa diff --git a/third_party/protobuf/ruby/.gitignore b/third_party/protobuf/ruby/.gitignore new file mode 100644 index 0000000000..bd8745dd85 --- /dev/null +++ b/third_party/protobuf/ruby/.gitignore @@ -0,0 +1,8 @@ +*.bundle +tags +.idea/ +lib/google/protobuf_java.jar +protobuf-jruby.iml +target/ +pkg/ +tmp/ diff --git a/tools_webrtc/autoroller/roll_deps.py b/tools_webrtc/autoroller/roll_deps.py index 469668b1c6..4dce62eb3c 100755 --- a/tools_webrtc/autoroller/roll_deps.py +++ b/tools_webrtc/autoroller/roll_deps.py @@ -19,22 +19,24 @@ import subprocess import sys import urllib2 - # Skip these dependencies (list without solution name prefix). DONT_AUTOROLL_THESE = [ 'src/examples/androidtests/third_party/gradle', + 'src/third_party_chromium', ] # Run these CQ trybots in addition to the default ones in infra/config/cq.cfg. EXTRA_TRYBOTS = ( - 'master.internal.tryserver.corp.webrtc:linux_internal' + 'master.internal.tryserver.corp.webrtc:linux_internal' ) WEBRTC_URL = 'https://webrtc.googlesource.com/src' CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src' +CHROMIUM_THIRD_PARTY_URL = '%s/third_party' % CHROMIUM_SRC_URL CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s' CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s' +CHROMIUM_3P_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/third_party/+log/%s' COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'(\d+)\'$') @@ -47,6 +49,7 @@ CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir)) sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) import find_depot_tools + find_depot_tools.add_depot_tools_to_path() CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py' @@ -60,6 +63,13 @@ CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages') ChangedCipdPackage = collections.namedtuple( 'ChangedCipdPackage', 'path package current_version new_version') +ChromiumRevisionUpdate = collections.namedtuple('ChromiumRevisionUpdate', + ('current_chromium_rev ' + 'new_chromium_rev ' + 'current_third_party_rev ' + 'new_third_party_rev')) + + class RollError(Exception): pass @@ -74,7 +84,7 @@ def ParseDepsDict(deps_content): 'Var': VarLookup(local_scope), 'deps_os': {}, } - exec(deps_content, global_scope, local_scope) + exec (deps_content, global_scope, local_scope) return local_scope @@ -100,7 +110,7 @@ def ParseCommitPosition(commit_message): def _RunCommand(command, working_dir=None, ignore_exit_code=False, - extra_env=None): + extra_env=None, input_data=None): """Runs a command and returns the output from that command. If the command fails (exit code != 0), the function will exit the process. @@ -115,12 +125,12 @@ def _RunCommand(command, working_dir=None, ignore_exit_code=False, assert all(type(value) == str for value in extra_env.values()) logging.debug('extra env: %s', extra_env) env.update(extra_env) - p = subprocess.Popen(command, stdout=subprocess.PIPE, + p = subprocess.Popen(command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, cwd=working_dir, universal_newlines=True) - std_output = p.stdout.read() - err_output = p.stderr.read() - p.wait() + std_output, err_output = p.communicate(input_data) p.stdout.close() p.stderr.close() if not ignore_exit_code and p.returncode != 0: @@ -209,6 +219,7 @@ def GetMatchingDepsEntries(depsentry_dict, dir_path): def BuildDepsentryDict(deps_dict): """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict.""" result = {} + def AddDepsEntries(deps_subdict): for path, dep in deps_subdict.iteritems(): if path in result: @@ -279,8 +290,8 @@ def CalculateChangedDeps(webrtc_deps, new_cr_deps): # Use the revision from Chromium's DEPS file. new_rev = cr_deps_entry.revision assert webrtc_deps_entry.url == cr_deps_entry.url, ( - 'WebRTC DEPS entry %s has a different URL (%s) than Chromium (%s).' % - (path, webrtc_deps_entry.url, cr_deps_entry.url)) + 'WebRTC DEPS entry %s has a different URL (%s) than Chromium (%s).' % + (path, webrtc_deps_entry.url, cr_deps_entry.url)) else: # Use the HEAD of the deps repo. stdout, _ = _RunCommand(['git', 'ls-remote', webrtc_deps_entry.url, @@ -308,23 +319,28 @@ def CalculateChangedClang(new_cr_rev): current_rev = GetClangRev(current_lines) new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH, - new_cr_rev).splitlines() + new_cr_rev).splitlines() new_rev = GetClangRev(new_clang_update_py) return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev) -def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos, - new_commit_pos, changed_deps_list, clang_change): - current_cr_rev = current_cr_rev[0:10] - new_cr_rev = new_cr_rev[0:10] +def GenerateCommitMessage(rev_update, current_commit_pos, + new_commit_pos, changed_deps_list, clang_change): + current_cr_rev = rev_update.current_chromium_rev[0:10] + new_cr_rev = rev_update.new_chromium_rev[0:10] rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) + rev_3p_interval = '%s..%s' % (rev_update.current_third_party_rev[0:10], + rev_update.new_third_party_rev[0:10]) git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval, - git_number_interval)] - commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval)) - commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % - rev_interval)) + git_number_interval), + 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval), + 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % + rev_interval), + 'Roll chromium third_party %s' % rev_3p_interval, + 'Change log: %s\n' % ( + CHROMIUM_3P_LOG_TEMPLATE % rev_3p_interval)] tbr_authors = '' if changed_deps_list: commit_msg.append('Changed dependencies:') @@ -365,14 +381,16 @@ def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos, return '\n'.join(commit_msg) -def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision, - changed_deps): +def UpdateDepsFile(deps_filename, rev_update, changed_deps): """Update the DEPS file with the new revision.""" # Update the chromium_revision variable. with open(deps_filename, 'rb') as deps_file: deps_content = deps_file.read() - deps_content = deps_content.replace(old_cr_revision, new_cr_revision) + deps_content = deps_content.replace(rev_update.current_chromium_rev, + rev_update.new_chromium_rev) + deps_content = deps_content.replace(rev_update.current_third_party_rev, + rev_update.new_third_party_rev) with open(deps_filename, 'wb') as deps_file: deps_file.write(deps_content) @@ -395,6 +413,36 @@ def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision, working_dir=CHECKOUT_SRC_DIR) +def _LoadThirdPartyDepsAndFiles(filename): + third_party_deps = {} + with open(filename, 'rb') as f: + deps_content = f.read() + global_scope = {} + exec (deps_content, global_scope, third_party_deps) + return third_party_deps.get('DEPS', []) + + +def UpdateThirdPartyDeps(new_rev, dest_dir, source_dir, + third_party_deps_file): + """Syncing deps, specified in third_party_deps_file with repo in source_dir. + + Will exit if sync failed for some reasons. + Params: + new_rev - revision of third_party to update to + dest_dir - webrtc directory, that will be used as root for third_party deps + source_dir - checked out chromium third_party repo + third_party_deps_file - file with list of third_party deps to copy + """ + + deps_to_checkout = _LoadThirdPartyDepsAndFiles(third_party_deps_file) + # Update existing chromium third_party checkout to new rev. + _RunCommand(['git', 'fetch', 'origin', new_rev], working_dir=source_dir) + # Checkout chromium repo into dest dir basing on source checkout. + _RunCommand( + ['git', '--git-dir', '%s/.git' % source_dir, 'checkout', + new_rev] + deps_to_checkout, working_dir=dest_dir) + + def _IsTreeClean(): stdout, _ = _RunCommand(['git', 'status', '--porcelain']) if len(stdout) == 0: @@ -437,6 +485,7 @@ def _LocalCommit(commit_msg, dry_run): logging.info('Committing changes locally.') if not dry_run: _RunCommand(['git', 'add', '--update', '.']) + _RunCommand(['git', 'add', '-A', 'third_party']) _RunCommand(['git', 'commit', '-m', commit_msg]) @@ -466,6 +515,30 @@ def _UploadCL(commit_queue_mode): _RunCommand(cmd, extra_env={'EDITOR': 'true', 'SKIP_GCE_AUTH_FOR_GIT': '1'}) +def GetRollRevisionRanges(opts, webrtc_deps): + current_cr_rev = webrtc_deps['vars']['chromium_revision'] + current_third_party_rev = webrtc_deps['vars']['chromium_third_party_revision'] + new_cr_rev = opts.revision + if not new_cr_rev: + stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) + head_rev = stdout.strip().split('\t')[0] + logging.info('No revision specified. Using HEAD: %s', head_rev) + new_cr_rev = head_rev + + new_third_party_rev = opts.third_party_revision + if not new_third_party_rev: + stdout, _ = _RunCommand( + ['git', 'ls-remote', CHROMIUM_THIRD_PARTY_URL, 'HEAD']) + new_third_party_rev = stdout.strip().split('\t')[0] + logging.info( + 'No third_party revision specified. Using HEAD: %s', + new_third_party_rev) + + return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev, + current_third_party_rev, + new_third_party_rev) + + def main(): p = argparse.ArgumentParser() p.add_argument('--clean', action='store_true', default=False, @@ -473,6 +546,9 @@ def main(): p.add_argument('-r', '--revision', help=('Chromium Git revision to roll to. Defaults to the ' 'Chromium HEAD revision if omitted.')) + p.add_argument('--third-party-revision', + help=('Chromium third_party Git revision to roll to. Default ' + 'to the Chromium third_party HEAD revision if omitted.')) p.add_argument('-u', '--rietveld-email', help=('E-mail address to use for creating the CL at Rietveld' 'If omitted a previously cached one will be used or an ' @@ -510,30 +586,38 @@ def main(): if not opts.ignore_unclean_workdir: _EnsureUpdatedMasterBranch(opts.dry_run) - new_cr_rev = opts.revision - if not new_cr_rev: - stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) - head_rev = stdout.strip().split('\t')[0] - logging.info('No revision specified. Using HEAD: %s', head_rev) - new_cr_rev = head_rev - deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') webrtc_deps = ParseLocalDepsFile(deps_filename) - current_cr_rev = webrtc_deps['vars']['chromium_revision'] - - current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev)) - new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev)) - - new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev) + cr_3p_repo = os.path.join(CHECKOUT_SRC_DIR, 'third_party_chromium') + if not os.path.exists(cr_3p_repo): + raise RollError('missing third_party_chromium/. ' + 'Please add this to your gclient: \n' + '"custom_vars": {\n' + ' "roll_chromium_into_webrtc": True,\n' + '},\n' + 'Then run "gclient sync" again.') + + rev_update = GetRollRevisionRanges(opts, webrtc_deps) + + current_commit_pos = ParseCommitPosition( + ReadRemoteCrCommit(rev_update.current_chromium_rev)) + new_commit_pos = ParseCommitPosition( + ReadRemoteCrCommit(rev_update.new_chromium_rev)) + + new_cr_deps = ParseRemoteCrDepsFile(rev_update.new_chromium_rev) changed_deps = CalculateChangedDeps(webrtc_deps, new_cr_deps) - clang_change = CalculateChangedClang(new_cr_rev) - commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev, + clang_change = CalculateChangedClang(rev_update.new_chromium_rev) + commit_msg = GenerateCommitMessage(rev_update, current_commit_pos, new_commit_pos, changed_deps, clang_change) logging.debug('Commit message:\n%s', commit_msg) _CreateRollBranch(opts.dry_run) - UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps) + UpdateThirdPartyDeps(rev_update.new_third_party_rev, + os.path.join(CHECKOUT_SRC_DIR, 'third_party'), + cr_3p_repo, + os.path.join(CHECKOUT_SRC_DIR, 'THIRD_PARTY_DEPS')) + UpdateDepsFile(deps_filename, rev_update, changed_deps) if _IsTreeClean(): logging.info("No DEPS changes detected, skipping CL creation.") else: diff --git a/tools_webrtc/autoroller/unittests/roll_deps_test.py b/tools_webrtc/autoroller/unittests/roll_deps_test.py index b6ac1be961..17535087fe 100755 --- a/tools_webrtc/autoroller/unittests/roll_deps_test.py +++ b/tools_webrtc/autoroller/unittests/roll_deps_test.py @@ -20,7 +20,8 @@ PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir) sys.path.append(PARENT_DIR) import roll_deps from roll_deps import CalculateChangedDeps, ChooseCQMode, \ - GetMatchingDepsEntries, ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile + GetMatchingDepsEntries, ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile, \ + ChromiumRevisionUpdate TEST_DATA_VARS = { @@ -90,7 +91,10 @@ class TestRollChromiumRevision(unittest.TestCase): new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111' current_rev = TEST_DATA_VARS['chromium_revision'] - UpdateDepsFile(self._webrtc_depsfile, current_rev, new_rev, []) + UpdateDepsFile(self._webrtc_depsfile, + ChromiumRevisionUpdate( + current_rev, new_rev, current_rev, new_rev), + []) with open(self._webrtc_depsfile) as deps_file: deps_contents = deps_file.read() self.assertTrue(new_rev in deps_contents, |